00010000:00010000:1.0:1788876618.967696:0:33302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000074b6be50/0x7d14dcdc37047650 lrc: 6/1,0 mode: PW/PW res: [0x2c0000401:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7636 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876618.967715:0:33302:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876618.967720:0:33302:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xc3:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876618.967722:0:33302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876618.967724:0:33302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876618.967727:0:33302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217f30 (ffffa06188721128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876618.967730:0:33302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876618.967732:0:33302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217f30 (ffffa061887216c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876618.967738:0:33302:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876618.967771:0:33302:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c2064cc0 00000008:00010000:1.0:1788876618.967803:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c2064cc0 00010000:00010000:1.0:1788876618.967823:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000074b6be50/0x7d14dcdc37047650 lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7636 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876618.967857:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898d500 - object 0x2c0000401:526 offset 0<>4096 00000100:00100000:1.0:1788876618.967888:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788876618.967904:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511074688:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876618.971770:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c2064cc0 00000020:00200000:1.0:1788876618.971785:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876618.971801:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c2064cc0 00000100:00100000:1.0:1788876618.971825:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511074688:192.168.203.111@tcp:3:grep.0 00000020:00010000:1.0:1788876619.115156:0:33307:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06190f665d8 (ffffa061887216c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876619.115162:0:33307:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190f665d8 (ffffa061887216c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.115165:0:33307:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d530 (ffffa06188721128/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876619.115177:0:33307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.115207:0:33307:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876619.115226:0:33307:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190f665d8 (ffffa061887216c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876619.115229:0:33307:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d530 (ffffa061887216c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876619.115238:0:33307:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876619.115249:0:33307:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc8:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876619.115254:0:33307:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-158: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876619.115259:0:33307:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1788876619.115738:0:33307:0:(file.c:479:ll_file_release()) START file f5.replay-single-158:[0x200001b71:0xc8:0x0](ffffa0619826aa08), flags 100000 00000002:00100000:1.0:1788876619.115800:0:33307:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa061889b9500 x1875771511077760/t68719477051(68719477051) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876635 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876619.115810:0:33307:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876619.115815:0:33307:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898d500 x1875771511078144/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876619.115836:0:33307:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898d500 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33307:1875771511078144:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876619.115903:0:33307:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190ed5840 going to sleep for 11 seconds 00000100:00100000:1.0:1788876619.226848:0:33309:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecc380 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33309:1875771511079424:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876619.226888:0:33309:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa061889bb100 x1875771511079040/t68719477055(68719477055) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876635 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1788876619.429576:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ecd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511081728:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876619.436126:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa06189722780 00000020:00200000:1.0:1788876619.436142:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876619.436165:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa06189722780 00000100:00100000:1.0:1788876619.436188:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511081728:192.168.203.111@tcp:3:grep.0 00010000:00010000:1.0:1788876619.518528:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876619.565544:0:33316:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190e33480 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33316:1875771511083520:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876619.565574:0:33316:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565614:0:33316:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565625:0:33316:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565639:0:33316:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565653:0:33316:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565669:0:33316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565678:0:33316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565685:0:33316:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876619.565718:0:33316:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa06190e33480 x1875771511083520/t68719477069(68719477069) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876635 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876619.565732:0:33316:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06190e33480 x1875771511083520/t68719477069(68719477069) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876635 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876619.565812:0:33316:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876619.565828:0:33316:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.565839:0:33316:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/209 (0 0) 00000080:00200000:1.0:1788876619.565846:0:33316:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876619.565888:0:33316:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xd1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788876619.565901:0:33316:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876619.565927:0:33316:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xd1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876619.566008:0:33316:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 0 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876619.566080:0:33316:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xd1:0x0](ffffa06198275b88): rc = 0 00000080:00200000:1.0:1788876619.566096:0:33316:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xd1:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876619.566222:0:33316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd1:0x0](ffffa06198275b88) for lock 0x7d14dcdc370480bb 00000080:00010000:1.0:1788876619.566237:0:33316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd1:0x0](ffffa06198275b88) 00000080:00200008:1.0:1788876619.566251:0:33316:0:(file.c:1090:ll_file_open()) START file f5.replay-single-167:[0x200001b71:0xd1:0x0](ffffa06198275b88), flags 100400 00000002:00100000:1.0:1788876619.566274:0:33316:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06190e33480 x1875771511083520/t68719477069(68719477069) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876635 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876619.566338:0:33316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370480bb from it ffffa061886dac00 00010000:00010000:1.0:1788876619.566343:0:33316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.566354:0:33316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876619.566376:0:33316:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xd1:0x0](ffffa06198275b88), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876619.566560:0:33316:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-167:[0x200001b71:0xd1:0x0](ffffa06198275b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876619.566568:0:33316:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xd1:0x0](ffffa06198275b88),name=f5.replay-single-167 00000002:00010000:1.0:1788876619.566586:0:33316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xd1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876619.566596:0:33316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876619.566608:0:33316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876619.566617:0:33316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876619.566622:0:33316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd1:0x0](ffffa06198275b88) 00000080:00010000:1.0:1788876619.566626:0:33316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd1:0x0](ffffa06198275b88) for lock 0x7d14dcdc370480bb 00000080:00010000:1.0:1788876619.566630:0:33316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370480bb from it ffffb95ac1fafcd8 00010000:00010000:1.0:1788876619.566633:0:33316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876619.566641:0:33316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ed800/0x7d14dcdc370480bb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab807 expref: -99 pid: 33316 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876619.566668:0:33316:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xd1:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876619.566676:0:33316:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd1:0x0] 00000008:00010000:1.0:1788876619.566688:0:33316:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876619.566692:0:33316:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217a30, osc lock ffffa061867931d0, flags 0x400001002 00010000:00010000:1.0:1788876619.566707:0:33316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876619.566718:0:33316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876619.566732:0:33316:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876619.566812:0:33316:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876619.566816:0:33316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876619.566824:0:33316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876619.566835:0:33316:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876619.566844:0:33316:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xd1:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876619.566848:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876619.566852:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.566855:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217a30 (ffffa06188721128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.566858:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876619.566868:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217a30 (ffffa061887216c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876619.566876:0:33316:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xd1:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876619.566892:0:33316:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-167:[0x200001b71:0xd1:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876619.566912:0:33316:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876619.566926:0:33316:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-167: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876619.566931:0:33316:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-167: desiginated mirror: 0 00000020:00200000:1.0:1788876619.566936:0:33316:0:(cl_io.c:214:cl_io_rw_init()) header@00000000f0840efb[0x110, 1, [0x200001b71:0xd1:0x0] hash] 00000020:00200000:1.0:1788876619.566938:0:33316:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876619.566941:0:33316:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xd1:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876619.566954:0:33316:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876619.566957:0:33316:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876619.566970:0:33316:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876619.566978:0:33316:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876619.566981:0:33316:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217a30, osc lock ffffa061867931d0, flags 0x0 00010000:00010000:1.0:1788876619.566989:0:33316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.566998:0:33316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.567006:0:33316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.567013:0:33316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.567020:0:33316:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 5/1,0 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876619.567054:0:33316:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-167: read [0, 32768) 00000001:00010000:1.0:1788876619.567065:0:33316:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd1:0x0] 00000008:00010000:1.0:1788876619.567071:0:33316:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876619.567073:0:33316:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217bb0, osc lock ffffa061867938f0, flags 0x400001002 00010000:00010000:1.0:1788876619.567081:0:33316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.567090:0:33316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876619.567099:0:33316:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876619.567102:0:33316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 7/1,1 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.567824:0:33316:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 6/1,0 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876619.567833:0:33316:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876619.567840:0:33316:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xd1:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876619.567843:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876619.567846:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.567849:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217bb0 (ffffa06188721128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.567852:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876619.567854:0:33316:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217bb0 (ffffa061887216c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876619.567863:0:33316:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876619.567903:0:33316:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c2064cc0 00000008:00010000:1.0:1788876619.567944:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c2064cc0 00010000:00010000:1.0:1788876619.567968:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876619.568007:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898ed80 - object 0x2c0000401:533 offset 0<>4096 00000100:00100000:1.0:1788876619.704261:0:33319:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecdf80 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33319:1875771511085440:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876619.704293:0:33319:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704317:0:33319:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704327:0:33319:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704341:0:33319:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704353:0:33319:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704368:0:33319:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704377:0:33319:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704417:0:33319:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876619.704445:0:33319:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa06190ecdf80 x1875771511085440/t68719477075(68719477075) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876635 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876619.704460:0:33319:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06190ecdf80 x1875771511085440/t68719477075(68719477075) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876635 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876619.704476:0:33319:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876619.704490:0:33319:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704502:0:33319:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/212 (0 0) 00000080:00200000:1.0:1788876619.704509:0:33319:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876619.704552:0:33319:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xd4:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876619.704565:0:33319:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876619.704591:0:33319:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xd4:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876619.704658:0:33319:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 0 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876619.704734:0:33319:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xd4:0x0](ffffa06198270908): rc = 0 00000080:00200000:1.0:1788876619.704751:0:33319:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xd4:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876619.704757:0:33319:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd4:0x0](ffffa06198270908) for lock 0x7d14dcdc370480d0 00000080:00010000:1.0:1788876619.704770:0:33319:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd4:0x0](ffffa06198270908) 00000080:00200008:1.0:1788876619.704787:0:33319:0:(file.c:1090:ll_file_open()) START file f5.replay-single-170:[0x200001b71:0xd4:0x0](ffffa06198270908), flags 100400 00000002:00100000:1.0:1788876619.704810:0:33319:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06190ecdf80 x1875771511085440/t68719477075(68719477075) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876635 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876619.704885:0:33319:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370480d0 from it ffffa061879fd240 00010000:00010000:1.0:1788876619.704890:0:33319:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876619.704902:0:33319:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876619.704930:0:33319:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xd4:0x0](ffffa06198270908), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876619.704957:0:33319:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-170:[0x200001b71:0xd4:0x0](ffffa06198270908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876619.704964:0:33319:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xd4:0x0](ffffa06198270908),name=f5.replay-single-170 00000002:00010000:1.0:1788876619.704982:0:33319:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xd4:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876619.704992:0:33319:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876619.705004:0:33319:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876619.705013:0:33319:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876619.705019:0:33319:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xd4:0x0](ffffa06198270908) 00000080:00010000:1.0:1788876619.705022:0:33319:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xd4:0x0](ffffa06198270908) for lock 0x7d14dcdc370480d0 00000080:00010000:1.0:1788876619.705026:0:33319:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370480d0 from it ffffb95ac1f77cd8 00010000:00010000:1.0:1788876619.705042:0:33319:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876619.705051:0:33319:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876619.705078:0:33319:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xd4:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876619.705087:0:33319:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd4:0x0] 00000008:00010000:1.0:1788876619.705103:0:33319:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876619.705107:0:33319:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217a30, osc lock ffffa061867938f0, flags 0x400001002 00010000:00010000:1.0:1788876619.705123:0:33319:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876619.705134:0:33319:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876619.705146:0:33319:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876619.705157:0:33319:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876619.705161:0:33319:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876619.705170:0:33319:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876619.705181:0:33319:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876619.705192:0:33319:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xd4:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876619.705196:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5bc0 (ffffa06188721a28/0) at cl_lock_release():245 00000020:00010000:1.0:1788876619.705200:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5bc0 (ffffa06188721a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.705202:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217a30 (ffffa06188721758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.705206:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5bc0 (ffffa06188721a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876619.705215:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217a30 (ffffa06188721a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876619.705223:0:33319:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xd4:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876619.705239:0:33319:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-170:[0x200001b71:0xd4:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876619.705261:0:33319:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876619.705277:0:33319:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-170: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876619.705282:0:33319:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-170: desiginated mirror: 0 00000020:00200000:1.0:1788876619.705288:0:33319:0:(cl_io.c:214:cl_io_rw_init()) header@00000000687cc8ae[0x110, 1, [0x200001b71:0xd4:0x0] hash] 00000020:00200000:1.0:1788876619.705290:0:33319:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876619.705293:0:33319:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xd4:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876619.705305:0:33319:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876619.705308:0:33319:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876619.705319:0:33319:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876619.705325:0:33319:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876619.705329:0:33319:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217a30, osc lock ffffa061867938f0, flags 0x0 00010000:00010000:1.0:1788876619.705338:0:33319:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.705348:0:33319:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.705357:0:33319:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.705365:0:33319:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.705372:0:33319:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 5/1,0 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876619.705430:0:33319:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-170: read [0, 32768) 00000001:00010000:1.0:1788876619.705441:0:33319:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xd4:0x0] 00000008:00010000:1.0:1788876619.705446:0:33319:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876619.705448:0:33319:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217930, osc lock ffffa06186793b50, flags 0x400001002 00010000:00010000:1.0:1788876619.705455:0:33319:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.705463:0:33319:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876619.705472:0:33319:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876619.705475:0:33319:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.705482:0:33319:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 6/1,0 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876619.705488:0:33319:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876619.705492:0:33319:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xd4:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876619.705494:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5bc0 (ffffa06188721a28/0) at cl_lock_release():245 00000020:00010000:1.0:1788876619.705497:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5bc0 (ffffa06188721a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.705500:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217930 (ffffa06188721758/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.705502:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5bc0 (ffffa06188721a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876619.705505:0:33319:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217930 (ffffa06188721a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876619.705511:0:33319:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876619.705549:0:33319:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c2064f00 00000008:00010000:1.0:1788876619.705595:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c2064f00 00010000:00010000:1.0:1788876619.705620:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b73f68e/0x7d14dcdc370477b5 lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a79ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876619.705668:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898f800 - object 0x280000401:534 offset 0<>4096 00000100:00100000:1.0:1788876619.705714:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511085568:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876619.714331:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c2064f00 00000020:00200000:1.0:1788876619.714345:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876619.714367:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c2064f00 00000100:00100000:1.0:1788876619.714430:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511085568:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876619.718770:0:33319:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecf800 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33319:1875771511085824:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876619.718811:0:33319:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06190ecdf80 x1875771511085440/t68719477075(68719477075) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876635 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00010000:1.0:1788876619.833475:0:33322:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06190f665d8 (ffffa061887216c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876619.833480:0:33322:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190f665d8 (ffffa061887216c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876619.833484:0:33322:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d830 (ffffa06188721128/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876619.833496:0:33322:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000028394c6c/0x7d14dcdc370477f4 lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7a96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876619.833512:0:33322:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000028394c6c/0x7d14dcdc370477f4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7a96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876619.833529:0:33322:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190f665d8 (ffffa061887216c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876619.833532:0:33322:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d830 (ffffa061887216c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876619.833540:0:33322:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876619.833550:0:33322:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xd7:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876619.833561:0:33322:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-173: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876619.833566:0:33322:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000100:00100000:1.0:1788876619.862791:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ecdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511088128:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876619.992261:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ecce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511090048:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876620.027727:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ecce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511090688:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876620.033079:0:2395:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa06189722780 00000020:00200000:1.0:1788876620.033094:0:2395:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876620.033141:0:2395:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa06189722780 00000100:00100000:1.0:1788876620.033167:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511090688:192.168.203.111@tcp:3:grep.0 00000080:00200000:1.0:1788876620.113999:0:33330:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876620.117119:0:33330:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.117149:0:33330:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 186 lru_type: 1 00010000:00010000:1.0:1788876620.117170:0:33330:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 186 lru_type: 1 00000080:00200000:1.0:1788876620.117179:0:33330:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.117192:0:33330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876620.117196:0:33330:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876620.117205:0:33330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac1fefbf8 00010000:00010000:1.0:1788876620.117209:0:33330:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 186 lru_type: 1 00010000:00010000:1.0:1788876620.117218:0:33330:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 187 lru_type: 1 00000080:00200000:1.0:1788876620.117257:0:33330:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876620.117277:0:33330:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-181, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa06184eb7c80, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1788876620.117283:0:33330:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f5.replay-single-181, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:1.0:1788876620.117303:0:33330:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f5.replay-single-181,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1788876620.117357:0:33330:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188995400/0x7d14dcdc3704811d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33330 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.117366:0:33330:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188995400/0x7d14dcdc3704811d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33330 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788876620.117520:0:33330:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788876620.117536:0:33330:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898f800 x1875771511092480/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788876620.117548:0:33330:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188995400/0x7d14dcdc3704811d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33330 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876620.117580:0:33330:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f800 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33330:1875771511092480:192.168.203.111@tcp:101:grep.0 00000100:00100000:1.0:1788876620.117973:0:33330:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876620.127297:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ece680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511092608:192.168.203.111@tcp:3:grep.0 00000020:00010000:1.0:1788876620.137671:0:33330:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618836ee48 (ffffa06182fa7a28/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.137678:0:33330:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618836ee48 (ffffa06182fa7a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.137683:0:33330:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061864d50b0 (ffffa06182fa76c8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876620.137696:0:33330:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000092d9cc70/0x7d14dcdc3704789c lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c56 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.137725:0:33330:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000092d9cc70/0x7d14dcdc3704789c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c56 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876620.137756:0:33330:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618836ee48 (ffffa06182fa7a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.137760:0:33330:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061864d50b0 (ffffa06182fa7a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.137769:0:33330:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876620.137782:0:33330:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xdf:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876620.137787:0:33330:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-181: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876620.137792:0:33330:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1788876620.139529:0:33330:0:(file.c:479:ll_file_release()) START file f5.replay-single-181:[0x200001b71:0xdf:0x0](ffffa06198267448), flags 100000 00000002:00100000:1.0:1788876620.139609:0:33330:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa0618898f800 x1875771511092480/t68719477097(68719477097) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876620.139623:0:33330:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876620.139629:0:33330:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898f100 x1875771511092864/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876620.139653:0:33330:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f100 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33330:1875771511092864:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876620.139787:0:33330:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000080:00200000:1.0:1788876620.168299:0:33331:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xe0:0x0](ffffa06198273a88): rc = 0 00000080:00200000:1.0:1788876620.168326:0:33331:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xe0:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876620.168334:0:33331:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe0:0x0](ffffa06198273a88) for lock 0x7d14dcdc37048124 00000080:00010000:1.0:1788876620.168346:0:33331:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe0:0x0](ffffa06198273a88) 00000080:00200008:1.0:1788876620.168361:0:33331:0:(file.c:1090:ll_file_open()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0](ffffa06198273a88), flags 100400 00000002:00100000:1.0:1788876620.168433:0:33331:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06190ece680 x1875771511093120/t68719477099(68719477099) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876620.168500:0:33331:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048124 from it ffffa061879fd840 00010000:00010000:1.0:1788876620.168507:0:33331:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188757e00/0x7d14dcdc37048124 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aba14 expref: -99 pid: 33331 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.168520:0:33331:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188757e00/0x7d14dcdc37048124 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aba14 expref: -99 pid: 33331 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.168545:0:33331:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xe0:0x0](ffffa06198273a88), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876620.168570:0:33331:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0](ffffa06198273a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876620.168577:0:33331:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe0:0x0](ffffa06198273a88),name=f5.replay-single-182 00000002:00010000:1.0:1788876620.168595:0:33331:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xe0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.168608:0:33331:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188757e00/0x7d14dcdc37048124 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aba14 expref: -99 pid: 33331 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.168621:0:33331:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188757e00/0x7d14dcdc37048124 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aba14 expref: -99 pid: 33331 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.168630:0:33331:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.168635:0:33331:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe0:0x0](ffffa06198273a88) 00000080:00010000:1.0:1788876620.168639:0:33331:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe0:0x0](ffffa06198273a88) for lock 0x7d14dcdc37048124 00000080:00010000:1.0:1788876620.168643:0:33331:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048124 from it ffffb95ac1f27cd8 00010000:00010000:1.0:1788876620.168646:0:33331:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188757e00/0x7d14dcdc37048124 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aba14 expref: -99 pid: 33331 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.168655:0:33331:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188757e00/0x7d14dcdc37048124 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aba14 expref: -99 pid: 33331 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.168682:0:33331:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876620.168691:0:33331:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe0:0x0] 00000008:00010000:1.0:1788876620.168705:0:33331:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.168708:0:33331:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217930, osc lock ffffa06186793b50, flags 0x400001002 00010000:00010000:1.0:1788876620.168725:0:33331:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.168737:0:33331:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876620.168751:0:33331:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876620.168761:0:33331:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.168764:0:33331:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.168772:0:33331:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.168782:0:33331:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.168792:0:33331:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xe0:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.168796:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.168800:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.168802:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217930 (ffffa06188721c68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.168806:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.168815:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217930 (ffffa06188721cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.168827:0:33331:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876620.168845:0:33331:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876620.168879:0:33331:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876620.168895:0:33331:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-182: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876620.168900:0:33331:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-182: desiginated mirror: 0 00000020:00200000:1.0:1788876620.168906:0:33331:0:(cl_io.c:214:cl_io_rw_init()) header@00000000d063ac35[0x110, 1, [0x200001b71:0xe0:0x0] hash] 00000020:00200000:1.0:1788876620.168908:0:33331:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876620.168910:0:33331:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.168924:0:33331:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876620.168928:0:33331:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.168940:0:33331:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876620.168948:0:33331:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876620.168952:0:33331:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217930, osc lock ffffa06186793b50, flags 0x0 00010000:00010000:1.0:1788876620.168961:0:33331:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.168970:0:33331:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.168979:0:33331:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.168987:0:33331:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.168994:0:33331:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 5/1,0 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876620.169013:0:33331:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-182: read [0, 32768) 00000001:00010000:1.0:1788876620.169022:0:33331:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe0:0x0] 00000008:00010000:1.0:1788876620.169039:0:33331:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.169041:0:33331:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217a30, osc lock ffffa06186793690, flags 0x400001002 00010000:00010000:1.0:1788876620.169049:0:33331:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.169058:0:33331:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.169065:0:33331:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.169068:0:33331:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.169075:0:33331:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 6/1,0 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.169080:0:33331:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.169084:0:33331:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xe0:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.169086:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.169088:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.169091:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217a30 (ffffa06188721c68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.169093:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.169095:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217a30 (ffffa06188721cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.169101:0:33331:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876620.169143:0:33331:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c2064f00 00000008:00010000:1.0:1788876620.169186:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c2064f00 00010000:00010000:1.0:1788876620.169207:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876620.169239:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898ea00 - object 0x280000401:540 offset 0<>4096 00000020:00010000:1.0:1788876620.173732:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06190f668a8 (ffffa06188721cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.173748:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190f668a8 (ffffa06188721cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.173753:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217930 (ffffa06188721c68/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876620.173764:0:33331:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.173780:0:33331:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007da5825e/0x7d14dcdc370478b1 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x21c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7c8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876620.173797:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190f668a8 (ffffa06188721cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.173800:0:33331:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217930 (ffffa06188721cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.173809:0:33331:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876620.173820:0:33331:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xe0:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876620.173825:0:33331:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-182: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876620.173829:0:33331:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1788876620.174568:0:33331:0:(file.c:479:ll_file_release()) START file f5.replay-single-182:[0x200001b71:0xe0:0x0](ffffa06198273a88), flags 100000 00000002:00100000:1.0:1788876620.174635:0:33331:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa06190ece680 x1875771511093120/t68719477099(68719477099) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876620.174646:0:33331:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876620.174651:0:33331:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898d180 x1875771511093504/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876620.174671:0:33331:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898d180 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33331:1875771511093504:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876620.174965:0:33331:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000080:00200000:1.0:1788876620.278283:0:33335:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876620.278313:0:33335:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.278334:0:33335:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 191 lru_type: 1 00010000:00010000:1.0:1788876620.278362:0:33335:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 191 lru_type: 1 00000080:00200000:1.0:1788876620.278371:0:33335:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.278480:0:33335:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876620.278485:0:33335:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876620.278492:0:33335:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac1fefbf8 00010000:00010000:1.0:1788876620.278496:0:33335:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 191 lru_type: 1 00010000:00010000:1.0:1788876620.278506:0:33335:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 192 lru_type: 1 00000080:00200000:1.0:1788876620.278524:0:33335:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876620.278538:0:33335:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-186, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa0618817dc80, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1788876620.278545:0:33335:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f5.replay-single-186, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:1.0:1788876620.278559:0:33335:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f5.replay-single-186,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1788876620.278613:0:33335:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994400/0x7d14dcdc37048140 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33335 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.278622:0:33335:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994400/0x7d14dcdc37048140 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33335 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788876620.278640:0:33335:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788876620.278651:0:33335:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898df80 x1875771511095680/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788876620.278660:0:33335:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994400/0x7d14dcdc37048140 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33335 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876620.278682:0:33335:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898df80 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33335:1875771511095680:192.168.203.111@tcp:101:grep.0 00000100:00100000:1.0:1788876620.278936:0:33335:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876620.408579:0:33338:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190e31f80 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33338:1875771511097984:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876620.408615:0:33338:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06190e31500 x1875771511097600/t68719477113(68719477113) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788876620.413709:0:33339:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876620.413739:0:33339:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.413760:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 195 lru_type: 1 00010000:00010000:1.0:1788876620.413777:0:33339:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 195 lru_type: 1 00000080:00200000:1.0:1788876620.413784:0:33339:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.413792:0:33339:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876620.413795:0:33339:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876620.413803:0:33339:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac201fbf8 00010000:00010000:1.0:1788876620.413807:0:33339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 195 lru_type: 1 00010000:00010000:1.0:1788876620.413827:0:33339:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 196 lru_type: 1 00000080:00200000:1.0:1788876620.413843:0:33339:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876620.413856:0:33339:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-190, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa061a03d2180, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1788876620.413863:0:33339:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f5.replay-single-190, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:1.0:1788876620.413878:0:33339:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f5.replay-single-190,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1788876620.413931:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33339 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.413939:0:33339:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33339 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788876620.413950:0:33339:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788876620.413962:0:33339:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511098240/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788876620.413971:0:33339:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33339 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876620.413991:0:33339:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33339:1875771511098240:192.168.203.111@tcp:101:grep.0 00000100:00100000:1.0:1788876620.415794:0:33339:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876620.419573:0:33339:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33339:1875771511098240:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876620.419609:0:33339:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419632:0:33339:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419658:0:33339:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419672:0:33339:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419684:0:33339:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419699:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419708:0:33339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419715:0:33339:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876620.419728:0:33339:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa0618898dc00 x1875771511098240/t68719477115(68719477115) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876620.419741:0:33339:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898dc00 x1875771511098240/t68719477115(68719477115) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876620.419756:0:33339:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876620.419768:0:33339:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.419788:0:33339:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/232 (0 0) 00000080:00200000:1.0:1788876620.419795:0:33339:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876620.419850:0:33339:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xe8:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.419862:0:33339:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.419891:0:33339:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xe8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876620.419957:0:33339:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876563 1788876563 0 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.420048:0:33339:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xe8:0x0](ffffa0619827cb08): rc = 0 00000080:00200000:1.0:1788876620.420066:0:33339:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xe8:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876620.420071:0:33339:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe8:0x0](ffffa0619827cb08) for lock 0x7d14dcdc3704815c 00000080:00010000:1.0:1788876620.420082:0:33339:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe8:0x0](ffffa0619827cb08) 00000080:00200008:1.0:1788876620.420096:0:33339:0:(file.c:1090:ll_file_open()) START file f5.replay-single-190:[0x200001b71:0xe8:0x0](ffffa0619827cb08), flags 100400 00000002:00100000:1.0:1788876620.420117:0:33339:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898dc00 x1875771511098240/t68719477115(68719477115) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876620.420169:0:33339:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704815c from it ffffa061888ab420 00010000:00010000:1.0:1788876620.420174:0:33339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.420184:0:33339:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.420207:0:33339:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xe8:0x0](ffffa0619827cb08), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876620.420230:0:33339:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-190:[0x200001b71:0xe8:0x0](ffffa0619827cb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876620.420237:0:33339:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe8:0x0](ffffa0619827cb08),name=f5.replay-single-190 00000002:00010000:1.0:1788876620.420258:0:33339:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xe8:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.420267:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.420279:0:33339:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.420286:0:33339:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.420291:0:33339:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe8:0x0](ffffa0619827cb08) 00000080:00010000:1.0:1788876620.420295:0:33339:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe8:0x0](ffffa0619827cb08) for lock 0x7d14dcdc3704815c 00000080:00010000:1.0:1788876620.420299:0:33339:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704815c from it ffffb95ac201fcd8 00010000:00010000:1.0:1788876620.420302:0:33339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.420309:0:33339:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc3704815c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb2c expref: -99 pid: 33339 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.420323:0:33339:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xe8:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876620.420330:0:33339:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe8:0x0] 00000008:00010000:1.0:1788876620.420344:0:33339:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.420347:0:33339:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217a30, osc lock ffffa06186793b50, flags 0x400001002 00010000:00010000:1.0:1788876620.420360:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.420369:0:33339:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876620.420410:0:33339:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876620.420426:0:33339:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.420430:0:33339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.420438:0:33339:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.420449:0:33339:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.420459:0:33339:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xe8:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.420463:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.420466:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.420469:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190217a30 (ffffa06188721c68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.420473:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.420475:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190217a30 (ffffa06188721cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.420482:0:33339:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xe8:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876620.420498:0:33339:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-190:[0x200001b71:0xe8:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876620.420519:0:33339:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876620.420534:0:33339:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-190: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876620.420539:0:33339:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-190: desiginated mirror: 0 00000020:00200000:1.0:1788876620.420544:0:33339:0:(cl_io.c:214:cl_io_rw_init()) header@000000004baa3a79[0x110, 1, [0x200001b71:0xe8:0x0] hash] 00000020:00200000:1.0:1788876620.420546:0:33339:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876620.420549:0:33339:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xe8:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.420560:0:33339:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876620.420563:0:33339:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.420575:0:33339:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876620.420581:0:33339:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876620.420584:0:33339:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190217a30, osc lock ffffa06186793b50, flags 0x0 00010000:00010000:1.0:1788876620.420598:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.420609:0:33339:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.420618:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.420626:0:33339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.420632:0:33339:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 5/1,0 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876620.420640:0:33339:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-190: read [0, 32768) 00000001:00010000:1.0:1788876620.420648:0:33339:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe8:0x0] 00000008:00010000:1.0:1788876620.420654:0:33339:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.420656:0:33339:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e64930, osc lock ffffa06186793690, flags 0x400001002 00010000:00010000:1.0:1788876620.420662:0:33339:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.420671:0:33339:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.420679:0:33339:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.420681:0:33339:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.420700:0:33339:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 6/1,0 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.420707:0:33339:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.420712:0:33339:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xe8:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.420714:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.420717:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.420719:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e64930 (ffffa06188721c68/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.420722:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a4250 (ffffa06188721cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.420724:0:33339:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e64930 (ffffa06188721cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.420730:0:33339:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876620.420768:0:33339:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c20649c0 00000008:00010000:1.0:1788876620.420803:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c20649c0 00010000:00010000:1.0:1788876620.420823:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005da349b9/0x7d14dcdc37047959 lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x220:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7e4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876620.420856:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898c380 - object 0x280000401:544 offset 0<>4096 00000008:00010000:1.0:1788876620.423992:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c20649c0 00000020:00200000:1.0:1788876620.424006:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876620.424070:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c20649c0 00000100:00100000:1.0:1788876620.424097:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511098368:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876620.432562:0:33339:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ece300 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33339:1875771511098624:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876620.432601:0:33339:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa0618898dc00 x1875771511098240/t68719477115(68719477115) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1788876620.476681:0:33341:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061889b8a80 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33341:1875771511099520:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876620.476722:0:33341:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476743:0:33341:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476751:0:33341:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476776:0:33341:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476788:0:33341:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476803:0:33341:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476812:0:33341:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476820:0:33341:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876620.476832:0:33341:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa061889b8a80 x1875771511099520/t68719477119(68719477119) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876620.476844:0:33341:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061889b8a80 x1875771511099520/t68719477119(68719477119) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876620.476863:0:33341:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876620.476875:0:33341:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.476886:0:33341:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/234 (0 0) 00000080:00200000:1.0:1788876620.476891:0:33341:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876620.476934:0:33341:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xea:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.476946:0:33341:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.476973:0:33341:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xea:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876620.477047:0:33341:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876563 1788876563 0 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.477109:0:33341:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xea:0x0](ffffa06198266c08): rc = 0 00000080:00200000:1.0:1788876620.477129:0:33341:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xea:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876620.477136:0:33341:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xea:0x0](ffffa06198266c08) for lock 0x7d14dcdc3704816a 00000080:00010000:1.0:1788876620.477159:0:33341:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xea:0x0](ffffa06198266c08) 00000080:00200008:1.0:1788876620.477174:0:33341:0:(file.c:1090:ll_file_open()) START file f5.replay-single-192:[0x200001b71:0xea:0x0](ffffa06198266c08), flags 100400 00000002:00100000:1.0:1788876620.477198:0:33341:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061889b8a80 x1875771511099520/t68719477119(68719477119) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876620.477252:0:33341:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704816a from it ffffa06188808ae0 00010000:00010000:1.0:1788876620.477257:0:33341:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.477276:0:33341:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.477298:0:33341:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xea:0x0](ffffa06198266c08), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876620.477334:0:33341:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-192:[0x200001b71:0xea:0x0](ffffa06198266c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876620.477342:0:33341:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xea:0x0](ffffa06198266c08),name=f5.replay-single-192 00000002:00010000:1.0:1788876620.477361:0:33341:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xea:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.477371:0:33341:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.477425:0:33341:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.477433:0:33341:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.477438:0:33341:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xea:0x0](ffffa06198266c08) 00000080:00010000:1.0:1788876620.477442:0:33341:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xea:0x0](ffffa06198266c08) for lock 0x7d14dcdc3704816a 00000080:00010000:1.0:1788876620.477446:0:33341:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704816a from it ffffb95ac2017cd8 00010000:00010000:1.0:1788876620.477449:0:33341:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.477457:0:33341:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840200/0x7d14dcdc3704816a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xea:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abb72 expref: -99 pid: 33341 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.477471:0:33341:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xea:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876620.477479:0:33341:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xea:0x0] 00000008:00010000:1.0:1788876620.477493:0:33341:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.477501:0:33341:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa061a1c64b30, osc lock ffffa06186793690, flags 0x400001002 00010000:00010000:1.0:1788876620.477517:0:33341:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.477528:0:33341:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876620.477541:0:33341:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876620.477552:0:33341:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.477556:0:33341:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.477564:0:33341:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.477575:0:33341:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.477584:0:33341:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xea:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.477588:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a46f0 (ffffa06182fa7a28/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.477591:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a46f0 (ffffa06182fa7a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.477594:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061a1c64b30 (ffffa06182fa76c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.477598:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a46f0 (ffffa06182fa7a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.477601:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061a1c64b30 (ffffa06182fa7a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.477610:0:33341:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xea:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876620.477626:0:33341:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-192:[0x200001b71:0xea:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876620.477647:0:33341:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876620.477666:0:33341:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-192: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876620.477670:0:33341:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-192: desiginated mirror: 0 00000020:00200000:1.0:1788876620.477676:0:33341:0:(cl_io.c:214:cl_io_rw_init()) header@000000001f1f255c[0x110, 1, [0x200001b71:0xea:0x0] hash] 00000020:00200000:1.0:1788876620.477678:0:33341:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876620.477680:0:33341:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xea:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.477691:0:33341:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876620.477694:0:33341:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.477706:0:33341:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876620.477712:0:33341:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876620.477716:0:33341:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa061a1c64b30, osc lock ffffa06186793690, flags 0x0 00010000:00010000:1.0:1788876620.477724:0:33341:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.477734:0:33341:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.477744:0:33341:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.477752:0:33341:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.477759:0:33341:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 5/1,0 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876620.477767:0:33341:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-192: read [0, 32768) 00000001:00010000:1.0:1788876620.477777:0:33341:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xea:0x0] 00000008:00010000:1.0:1788876620.477796:0:33341:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.477799:0:33341:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2730, osc lock ffffa06186792980, flags 0x400001002 00010000:00010000:1.0:1788876620.477807:0:33341:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.477821:0:33341:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.477830:0:33341:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.477834:0:33341:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.477841:0:33341:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 6/1,0 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.477848:0:33341:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.477854:0:33341:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xea:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.477857:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a46f0 (ffffa06182fa7a28/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.477859:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a46f0 (ffffa06182fa7a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.477862:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2730 (ffffa06182fa76c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.477865:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a46f0 (ffffa06182fa7a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.477868:0:33341:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2730 (ffffa06182fa7a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.477874:0:33341:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876620.477910:0:33341:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c20649c0 00000008:00010000:1.0:1788876620.477945:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c20649c0 00010000:00010000:1.0:1788876620.477966:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876620.478000:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898dc00 - object 0x280000401:545 offset 0<>4096 00000100:00100000:1.0:1788876620.478057:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788876620.478075:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511099648:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876620.486521:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c20649c0 00000020:00200000:1.0:1788876620.486540:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876620.486561:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c20649c0 00000100:00100000:1.0:1788876620.486588:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511099648:192.168.203.111@tcp:3:grep.0 00000080:00200000:1.0:1788876620.573007:0:33345:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000080:00200000:1.0:1788876620.669227:0:33348:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876620.669266:0:33348:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.669288:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 204 lru_type: 1 00010000:00010000:1.0:1788876620.669306:0:33348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 204 lru_type: 1 00000080:00200000:1.0:1788876620.669315:0:33348:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.669323:0:33348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876620.669327:0:33348:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876620.669335:0:33348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac20a7bf8 00010000:00010000:1.0:1788876620.669339:0:33348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 204 lru_type: 1 00010000:00010000:1.0:1788876620.669347:0:33348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 205 lru_type: 1 00000080:00200000:1.0:1788876620.669364:0:33348:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876620.670331:0:33348:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-199, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa061a03d2900, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1788876620.670347:0:33348:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f5.replay-single-199, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:1.0:1788876620.670368:0:33348:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f5.replay-single-199,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1788876620.670469:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33348 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.670478:0:33348:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33348 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788876620.670491:0:33348:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788876620.670505:0:33348:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511104000/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788876620.670514:0:33348:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33348 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876620.670537:0:33348:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33348:1875771511104000:192.168.203.111@tcp:101:grep.0 00000100:00100000:1.0:1788876620.670724:0:33348:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64480 going to sleep for 11 seconds 00000100:00100000:1.0:1788876620.673977:0:33348:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33348:1875771511104000:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876620.674007:0:33348:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674039:0:33348:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674048:0:33348:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674061:0:33348:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674072:0:33348:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674101:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674110:0:33348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674118:0:33348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876620.674130:0:33348:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa0618898dc00 x1875771511104000/t68719477133(68719477133) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876620.674142:0:33348:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898dc00 x1875771511104000/t68719477133(68719477133) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876620.674155:0:33348:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876620.674169:0:33348:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674181:0:33348:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/241 (0 0) 00000080:00200000:1.0:1788876620.674188:0:33348:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876620.674243:0:33348:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf1:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788876620.674257:0:33348:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.674289:0:33348:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xf1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876620.674359:0:33348:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876563 1788876563 0 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.674468:0:33348:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xf1:0x0](ffffa0619827b248): rc = 0 00000080:00200000:1.0:1788876620.674494:0:33348:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xf1:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876620.674499:0:33348:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf1:0x0](ffffa0619827b248) for lock 0x7d14dcdc3704819b 00000080:00010000:1.0:1788876620.674510:0:33348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf1:0x0](ffffa0619827b248) 00000080:00200008:1.0:1788876620.674523:0:33348:0:(file.c:1090:ll_file_open()) START file f5.replay-single-199:[0x200001b71:0xf1:0x0](ffffa0619827b248), flags 100400 00000002:00100000:1.0:1788876620.674546:0:33348:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898dc00 x1875771511104000/t68719477133(68719477133) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876620.674602:0:33348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704819b from it ffffa061888ab420 00010000:00010000:1.0:1788876620.674607:0:33348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.674616:0:33348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.674641:0:33348:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xf1:0x0](ffffa0619827b248), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876620.674665:0:33348:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-199:[0x200001b71:0xf1:0x0](ffffa0619827b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876620.674672:0:33348:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xf1:0x0](ffffa0619827b248),name=f5.replay-single-199 00000002:00010000:1.0:1788876620.674692:0:33348:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xf1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.674703:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.674714:0:33348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.674720:0:33348:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.674733:0:33348:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf1:0x0](ffffa0619827b248) 00000080:00010000:1.0:1788876620.674736:0:33348:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf1:0x0](ffffa0619827b248) for lock 0x7d14dcdc3704819b 00000080:00010000:1.0:1788876620.674742:0:33348:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704819b from it ffffb95ac20a7cd8 00010000:00010000:1.0:1788876620.674745:0:33348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.674753:0:33348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704819b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc67 expref: -99 pid: 33348 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.674768:0:33348:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876620.674775:0:33348:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf1:0x0] 00000008:00010000:1.0:1788876620.674787:0:33348:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.674790:0:33348:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2630, osc lock ffffa06186792980, flags 0x400001002 00010000:00010000:1.0:1788876620.674806:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.674817:0:33348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876620.674829:0:33348:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876620.674842:0:33348:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.674845:0:33348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.674852:0:33348:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.674874:0:33348:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.674886:0:33348:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xf1:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.674890:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.674893:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.674896:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2630 (ffffa06188721128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.674899:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.674902:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2630 (ffffa061887216c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.674910:0:33348:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xf1:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876620.674925:0:33348:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-199:[0x200001b71:0xf1:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876620.674947:0:33348:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876620.674963:0:33348:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-199: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876620.674967:0:33348:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-199: desiginated mirror: 0 00000020:00200000:1.0:1788876620.674973:0:33348:0:(cl_io.c:214:cl_io_rw_init()) header@000000006e122461[0x110, 1, [0x200001b71:0xf1:0x0] hash] 00000020:00200000:1.0:1788876620.674975:0:33348:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876620.674978:0:33348:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf1:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.674989:0:33348:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876620.674992:0:33348:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.675001:0:33348:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876620.675008:0:33348:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876620.675011:0:33348:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2630, osc lock ffffa06186792980, flags 0x0 00010000:00010000:1.0:1788876620.675031:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.675041:0:33348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.675051:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.675065:0:33348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.675073:0:33348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 5/1,0 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876620.675082:0:33348:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-199: read [0, 32768) 00000001:00010000:1.0:1788876620.675093:0:33348:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf1:0x0] 00000008:00010000:1.0:1788876620.675097:0:33348:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.675100:0:33348:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2030, osc lock ffffa06186793430, flags 0x400001002 00010000:00010000:1.0:1788876620.675108:0:33348:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.675116:0:33348:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.675124:0:33348:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.675127:0:33348:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 7/1,1 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.675134:0:33348:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 6/1,0 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.675140:0:33348:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.675143:0:33348:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xf1:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.675146:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.675153:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.675157:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2030 (ffffa06188721128/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.675160:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a6e40 (ffffa061887216c8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.675163:0:33348:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2030 (ffffa061887216c8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.675169:0:33348:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876620.675208:0:33348:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c20649c0 00000008:00010000:1.0:1788876620.675249:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c20649c0 00010000:00010000:1.0:1788876620.675270:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000adec788d/0x7d14dcdc37047a16 lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8046 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876620.675306:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898c380 - object 0x2c0000401:549 offset 0<>4096 00000008:00010000:1.0:1788876620.678328:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c20649c0 00000020:00200000:1.0:1788876620.678345:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876620.678432:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c20649c0 00000100:00100000:1.0:1788876620.678458:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511104128:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876620.691778:0:33348:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecf100 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33348:1875771511104384:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876620.691825:0:33348:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa0618898dc00 x1875771511104000/t68719477133(68719477133) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788876620.697820:0:33349:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876620.697852:0:33349:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.697875:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 205 lru_type: 1 00010000:00010000:1.0:1788876620.697892:0:33349:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 205 lru_type: 1 00000080:00200000:1.0:1788876620.697902:0:33349:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.697916:0:33349:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876620.697920:0:33349:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876620.697929:0:33349:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac20afbf8 00010000:00010000:1.0:1788876620.697933:0:33349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 205 lru_type: 1 00010000:00010000:1.0:1788876620.697950:0:33349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 206 lru_type: 1 00000080:00200000:1.0:1788876620.697969:0:33349:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876620.697983:0:33349:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-200, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa06192f02780, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1788876620.697989:0:33349:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f5.replay-single-200, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:1.0:1788876620.698005:0:33349:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f5.replay-single-200,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1788876620.698075:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33349 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.698084:0:33349:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33349 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788876620.698098:0:33349:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788876620.698115:0:33349:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511104640/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788876620.698125:0:33349:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33349 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876620.698148:0:33349:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33349:1875771511104640:192.168.203.111@tcp:101:grep.0 00000100:00100000:1.0:1788876620.698443:0:33349:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64780 going to sleep for 11 seconds 00000100:00100000:1.0:1788876620.708205:0:33349:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33349:1875771511104640:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876620.708244:0:33349:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708268:0:33349:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708276:0:33349:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708288:0:33349:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708299:0:33349:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708313:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708322:0:33349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708331:0:33349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876620.708474:0:33349:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa0618898dc00 x1875771511104640/t68719477135(68719477135) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876620.708551:0:33349:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898dc00 x1875771511104640/t68719477135(68719477135) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876620.708624:0:33349:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876620.708638:0:33349:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708650:0:33349:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/242 (0 0) 00000080:00200000:1.0:1788876620.708657:0:33349:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876620.708698:0:33349:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf2:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.708710:0:33349:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.708735:0:33349:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xf2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876620.708787:0:33349:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876563 1788876563 0 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.708856:0:33349:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xf2:0x0](ffffa0619827ec08): rc = 0 00000080:00200000:1.0:1788876620.708873:0:33349:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xf2:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876620.708879:0:33349:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf2:0x0](ffffa0619827ec08) for lock 0x7d14dcdc370481a2 00000080:00010000:1.0:1788876620.708889:0:33349:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf2:0x0](ffffa0619827ec08) 00000080:00200008:1.0:1788876620.708900:0:33349:0:(file.c:1090:ll_file_open()) START file f5.replay-single-200:[0x200001b71:0xf2:0x0](ffffa0619827ec08), flags 100400 00000002:00100000:1.0:1788876620.708922:0:33349:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898dc00 x1875771511104640/t68719477135(68719477135) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876620.708975:0:33349:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370481a2 from it ffffa0618908c240 00010000:00010000:1.0:1788876620.708980:0:33349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.708999:0:33349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.709037:0:33349:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xf2:0x0](ffffa0619827ec08), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876620.709063:0:33349:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-200:[0x200001b71:0xf2:0x0](ffffa0619827ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876620.709069:0:33349:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xf2:0x0](ffffa0619827ec08),name=f5.replay-single-200 00000002:00010000:1.0:1788876620.709087:0:33349:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xf2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.709097:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.709110:0:33349:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.709117:0:33349:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.709121:0:33349:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf2:0x0](ffffa0619827ec08) 00000080:00010000:1.0:1788876620.709125:0:33349:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf2:0x0](ffffa0619827ec08) for lock 0x7d14dcdc370481a2 00000080:00010000:1.0:1788876620.709129:0:33349:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370481a2 from it ffffb95ac20afcd8 00010000:00010000:1.0:1788876620.709132:0:33349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.709140:0:33349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc370481a2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abc8a expref: -99 pid: 33349 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.709156:0:33349:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876620.709162:0:33349:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf2:0x0] 00000008:00010000:1.0:1788876620.709175:0:33349:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.709183:0:33349:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2fb0, osc lock ffffa06186793430, flags 0x400001002 00010000:00010000:1.0:1788876620.709199:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.709210:0:33349:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876620.709224:0:33349:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 4/0,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876620.709235:0:33349:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.709238:0:33349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.709247:0:33349:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.709256:0:33349:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.709265:0:33349:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xf2:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.709269:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a4940 (ffffa06182fa7908/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.709272:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a4940 (ffffa06182fa7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.709275:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2fb0 (ffffa06182fa7488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.709280:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a4940 (ffffa06182fa7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.709283:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2fb0 (ffffa06182fa7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.709291:0:33349:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xf2:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876620.709305:0:33349:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-200:[0x200001b71:0xf2:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876620.709326:0:33349:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876620.709356:0:33349:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-200: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876620.709360:0:33349:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-200: desiginated mirror: 0 00000020:00200000:1.0:1788876620.709366:0:33349:0:(cl_io.c:214:cl_io_rw_init()) header@000000005a2cc73b[0x110, 1, [0x200001b71:0xf2:0x0] hash] 00000020:00200000:1.0:1788876620.709369:0:33349:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876620.709372:0:33349:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf2:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.709403:0:33349:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876620.709407:0:33349:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.709418:0:33349:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876620.709425:0:33349:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876620.709428:0:33349:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2fb0, osc lock ffffa06186793430, flags 0x0 00010000:00010000:1.0:1788876620.709438:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.709449:0:33349:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.709458:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.709467:0:33349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 6/1,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.709476:0:33349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 5/1,0 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876620.711864:0:33349:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-200: read [0, 32768) 00000001:00010000:1.0:1788876620.711889:0:33349:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf2:0x0] 00000008:00010000:1.0:1788876620.711905:0:33349:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.711909:0:33349:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb26b0, osc lock ffffa061867930a0, flags 0x400001002 00010000:00010000:1.0:1788876620.711929:0:33349:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.711961:0:33349:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 4/1,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.711975:0:33349:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.711979:0:33349:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.711987:0:33349:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 6/1,0 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.711994:0:33349:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.712000:0:33349:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xf2:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.712004:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a4940 (ffffa06182fa7908/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.712007:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a4940 (ffffa06182fa7908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.712010:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb26b0 (ffffa06182fa7488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.712013:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a4940 (ffffa06182fa7908/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.712016:0:33349:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb26b0 (ffffa06182fa7908/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.712462:0:33349:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876620.712826:0:33349:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c20649c0 00000008:00010000:1.0:1788876620.712863:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c20649c0 00010000:00010000:1.0:1788876620.712886:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007d359ed2/0x7d14dcdc37047a2b lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x225:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a807e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876620.712927:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898c380 - object 0x280000401:549 offset 0<>4096 00000008:00010000:1.0:1788876620.719670:0:2395:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c20649c0 00000020:00200000:1.0:1788876620.719686:0:2395:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876620.719710:0:2395:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c20649c0 00000100:00100000:1.0:1788876620.719744:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511104768:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876620.723269:0:33349:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecf100 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33349:1875771511105024:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876620.723306:0:33349:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa0618898dc00 x1875771511104640/t68719477135(68719477135) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00010000:1.0:1788876620.770666:0:33350:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c54e48 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.770678:0:33350:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c54e48 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.770683:0:33350:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dcb0 (ffffa061890323f8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876620.770695:0:33350:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d5bd3155/0x7d14dcdc37047a40 lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a80b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.770713:0:33350:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d5bd3155/0x7d14dcdc37047a40 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a80b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876620.770732:0:33350:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c54e48 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.770735:0:33350:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dcb0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.770745:0:33350:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876620.770758:0:33350:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xf3:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876620.770762:0:33350:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-201: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876620.770767:0:33350:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1788876620.771243:0:33350:0:(file.c:479:ll_file_release()) START file f5.replay-single-201:[0x200001b71:0xf3:0x0](ffffa0619827ba88), flags 100000 00000002:00100000:1.0:1788876620.771475:0:33350:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa06190e30380 x1875771511105280/t68719477137(68719477137) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876620.771551:0:33350:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876620.771559:0:33350:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511105664/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876620.771594:0:33350:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33350:1875771511105664:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876620.771676:0:33350:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876620.779002:0:33350:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33350:1875771511105664:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876620.779056:0:33350:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06190e30380 x1875771511105280/t68719477137(68719477137) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788876620.784885:0:33351:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876620.784913:0:33351:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.784932:0:33351:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 207 lru_type: 1 00010000:00010000:1.0:1788876620.784950:0:33351:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 207 lru_type: 1 00000080:00200000:1.0:1788876620.784959:0:33351:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.784968:0:33351:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876620.784971:0:33351:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876620.784979:0:33351:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac20bfbf8 00010000:00010000:1.0:1788876620.784983:0:33351:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 207 lru_type: 1 00010000:00010000:1.0:1788876620.784992:0:33351:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 208 lru_type: 1 00000080:00200000:1.0:1788876620.785008:0:33351:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876620.791715:0:33351:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-202, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa0618817d080, open_flags 8100, mode 0 opened 0 00000080:00200008:1.0:1788876620.805978:0:33351:0:(file.c:479:ll_file_release()) START file f5.replay-single-202:[0x200001b71:0xf4:0x0](ffffa061982780c8), flags 100000 00000002:00100000:1.0:1788876620.813264:0:33351:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa06190ecfb80 x1875771511105920/t68719477139(68719477139) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876620.813282:0:33351:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876620.813289:0:33351:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511106304/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876620.813313:0:33351:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33351:1875771511106304:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876620.813419:0:33351:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876620.816601:0:33351:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33351:1875771511106304:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876620.816639:0:33351:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06190ecfb80 x1875771511105920/t68719477139(68719477139) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788876620.825690:0:33352:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876620.825720:0:33352:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.825805:0:33352:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 208 lru_type: 1 00010000:00010000:1.0:1788876620.825823:0:33352:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 208 lru_type: 1 00000080:00200000:1.0:1788876620.825832:0:33352:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.825841:0:33352:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876620.825845:0:33352:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876620.825852:0:33352:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac1fefbf8 00010000:00010000:1.0:1788876620.825856:0:33352:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 208 lru_type: 1 00010000:00010000:1.0:1788876620.825937:0:33352:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 209 lru_type: 1 00000080:00200000:1.0:1788876620.825954:0:33352:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876620.825967:0:33352:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-203, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa061a03d3980, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1788876620.825973:0:33352:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f5.replay-single-203, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:1.0:1788876620.825988:0:33352:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f5.replay-single-203,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1788876620.826049:0:33352:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f800/0x7d14dcdc370481b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33352 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.826058:0:33352:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f800/0x7d14dcdc370481b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33352 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788876620.826069:0:33352:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788876620.826080:0:33352:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511106560/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788876620.826089:0:33352:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f800/0x7d14dcdc370481b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33352 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876620.826110:0:33352:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33352:1875771511106560:192.168.203.111@tcp:101:grep.0 00000100:00100000:1.0:1788876620.826330:0:33352:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000008:00010000:1.0:1788876620.837016:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061897229c0 00000020:00200000:1.0:1788876620.837042:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876620.837089:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061897229c0 00000100:00100000:1.0:1788876620.837112:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511106688:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876620.919004:0:33356:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecc000 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33356:1875771511109120:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876620.919057:0:33356:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919078:0:33356:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919086:0:33356:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919099:0:33356:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919110:0:33356:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919123:0:33356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919131:0:33356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919139:0:33356:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876620.919150:0:33356:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa06190ecc000 x1875771511109120/t68719477149(68719477149) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876620.919163:0:33356:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06190ecc000 x1875771511109120/t68719477149(68719477149) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876620.919182:0:33356:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876620.919194:0:33356:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.919206:0:33356:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/249 (0 0) 00000080:00200000:1.0:1788876620.919212:0:33356:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876620.919253:0:33356:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf9:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788876620.919264:0:33356:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.919294:0:33356:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xf9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876620.919357:0:33356:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876564 1788876564 0 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.923666:0:33356:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0xf9:0x0](ffffa061981ba1c8): rc = 0 00000080:00200000:1.0:1788876620.923698:0:33356:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0xf9:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876620.923706:0:33356:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf9:0x0](ffffa061981ba1c8) for lock 0x7d14dcdc370481d3 00000080:00010000:1.0:1788876620.923718:0:33356:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf9:0x0](ffffa061981ba1c8) 00000080:00200008:1.0:1788876620.923846:0:33356:0:(file.c:1090:ll_file_open()) START file f5.replay-single-207:[0x200001b71:0xf9:0x0](ffffa061981ba1c8), flags 100400 00000002:00100000:1.0:1788876620.923937:0:33356:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06190ecc000 x1875771511109120/t68719477149(68719477149) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876620.924171:0:33356:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370481d3 from it ffffa06188816c60 00010000:00010000:1.0:1788876620.924180:0:33356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876620.924209:0:33356:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.924236:0:33356:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0xf9:0x0](ffffa061981ba1c8), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876620.924261:0:33356:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-207:[0x200001b71:0xf9:0x0](ffffa061981ba1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876620.924268:0:33356:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xf9:0x0](ffffa061981ba1c8),name=f5.replay-single-207 00000002:00010000:1.0:1788876620.924287:0:33356:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xf9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876620.924303:0:33356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.924315:0:33356:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876620.924324:0:33356:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876620.924330:0:33356:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xf9:0x0](ffffa061981ba1c8) 00000080:00010000:1.0:1788876620.924333:0:33356:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xf9:0x0](ffffa061981ba1c8) for lock 0x7d14dcdc370481d3 00000080:00010000:1.0:1788876620.924338:0:33356:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370481d3 from it ffffb95ac20afcd8 00010000:00010000:1.0:1788876620.924341:0:33356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876620.924349:0:33356:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf400/0x7d14dcdc370481d3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xf9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abd7f expref: -99 pid: 33356 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876620.924367:0:33356:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf9:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876620.924378:0:33356:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf9:0x0] 00000008:00010000:1.0:1788876620.924420:0:33356:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.924429:0:33356:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb22b0, osc lock ffffa061867930a0, flags 0x400001002 00010000:00010000:1.0:1788876620.924449:0:33356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.924461:0:33356:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876620.924476:0:33356:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876620.924485:0:33356:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.924489:0:33356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876620.924496:0:33356:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.924506:0:33356:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.924517:0:33356:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xf9:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.924521:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.924525:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.924528:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb22b0 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.924531:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.924533:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb22b0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.924544:0:33356:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xf9:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876620.924561:0:33356:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-207:[0x200001b71:0xf9:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876620.924585:0:33356:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876620.924605:0:33356:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-207: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876620.924610:0:33356:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-207: desiginated mirror: 0 00000020:00200000:1.0:1788876620.924616:0:33356:0:(cl_io.c:214:cl_io_rw_init()) header@000000007c5406e8[0x110, 1, [0x200001b71:0xf9:0x0] hash] 00000020:00200000:1.0:1788876620.924618:0:33356:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876620.924620:0:33356:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xf9:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876620.924631:0:33356:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876620.924635:0:33356:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876620.924648:0:33356:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876620.924656:0:33356:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876620.924659:0:33356:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb22b0, osc lock ffffa061867930a0, flags 0x0 00010000:00010000:1.0:1788876620.924668:0:33356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.924678:0:33356:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.924687:0:33356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.924695:0:33356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.924702:0:33356:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 5/1,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876620.924711:0:33356:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-207: read [0, 32768) 00000001:00010000:1.0:1788876620.924721:0:33356:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xf9:0x0] 00000008:00010000:1.0:1788876620.924726:0:33356:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876620.924728:0:33356:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb25b0, osc lock ffffa06186792850, flags 0x400001002 00010000:00010000:1.0:1788876620.924812:0:33356:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.924829:0:33356:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.924838:0:33356:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876620.924841:0:33356:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 7/1,1 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876620.924849:0:33356:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 6/1,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876620.924855:0:33356:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876620.924925:0:33356:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xf9:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876620.924929:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876620.924931:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.924934:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb25b0 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876620.924938:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876620.924940:0:33356:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb25b0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876620.924946:0:33356:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876620.924987:0:33356:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c20649c0 00000008:00010000:1.0:1788876620.925031:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c20649c0 00010000:00010000:1.0:1788876620.925052:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876620.925091:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898c380 - object 0x2c0000401:553 offset 0<>4096 00000100:00100000:1.0:1788876620.925135:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788876620.925154:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511109248:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876620.928352:0:2396:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c20649c0 00000020:00200000:1.0:1788876620.928365:0:2396:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876620.928515:0:2396:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c20649c0 00000100:00100000:1.0:1788876620.928535:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511109248:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876620.935268:0:33356:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecf480 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33356:1875771511109760:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876620.935314:0:33356:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06190ecc000 x1875771511109120/t68719477149(68719477149) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876636 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1788876621.033647:0:33361:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061995d4300 going to sleep for 11 seconds 00000020:00010000:1.0:1788876621.047888:0:33361:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06190f667b8 (ffffa06188721b48/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.047895:0:33361:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190f667b8 (ffffa06188721b48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.047899:0:33361:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061864d5ab0 (ffffa06188721908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876621.047910:0:33361:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.047926:0:33361:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876621.048032:0:33361:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190f667b8 (ffffa06188721b48/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.048036:0:33361:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061864d5ab0 (ffffa06188721b48/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.048049:0:33361:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876621.048063:0:33361:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xfd:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.048067:0:33361:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-211: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876621.048073:0:33361:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1788876621.048532:0:33361:0:(file.c:479:ll_file_release()) START file f5.replay-single-211:[0x200001b71:0xfd:0x0](ffffa061981bdb88), flags 100000 00000002:00100000:1.0:1788876621.048617:0:33361:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa061889b9500 x1875771511111936/t68719477157(68719477157) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876637 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876621.048642:0:33361:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876621.048648:0:33361:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898c380 x1875771511112320/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876621.048674:0:33361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33361:1875771511112320:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876621.048769:0:33361:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000020:00010000:1.0:1788876621.074804:0:33362:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c54c68 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.074811:0:33362:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c54c68 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.074816:0:33362:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d730 (ffffa06189032098/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876621.074826:0:33362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 4/1,0 mode: PW/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.074842:0:33362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876621.074867:0:33362:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c54c68 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.074870:0:33362:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d730 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.074878:0:33362:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876621.074892:0:33362:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xfe:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.074897:0:33362:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-212: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876621.074904:0:33362:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1788876621.075448:0:33362:0:(file.c:479:ll_file_release()) START file f5.replay-single-212:[0x200001b71:0xfe:0x0](ffffa0619827db88), flags 100000 00000002:00100000:1.0:1788876621.075513:0:33362:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa061889b9180 x1875771511112576/t68719477159(68719477159) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876637 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876621.075524:0:33362:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876621.075529:0:33362:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898c380 x1875771511112960/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876621.075557:0:33362:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33362:1875771511112960:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876621.075720:0:33362:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000020:00010000:1.0:1788876621.093803:0:33363:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06190f66038 (ffffa06188721488/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.093808:0:33363:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190f66038 (ffffa06188721488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.093812:0:33363:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190fe7a30 (ffffa06188721098/1) at cl_lock_cancel():142 00010000:00010000:1.0:1788876621.093821:0:33363:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.093835:0:33363:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:1.0:1788876621.093850:0:33363:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190f66038 (ffffa06188721488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.093853:0:33363:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190fe7a30 (ffffa06188721488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.093872:0:33363:0:(vvp_io.c:533:vvp_io_advance()) advancing 8 bytes 00000080:00200000:1.0:1788876621.093883:0:33363:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xff:0x0] ignore/verify layout 0/0, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.093888:0:33363:0:(file.c:2238:ll_file_io_generic()) f5.replay-single-213: 1 io complete with rc: 0, result: 8, restart: 0 00000080:00200000:1.0:1788876621.093893:0:33363:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 8 00000080:00200008:1.0:1788876621.094343:0:33363:0:(file.c:479:ll_file_release()) START file f5.replay-single-213:[0x200001b71:0xff:0x0](ffffa0619827a1c8), flags 100000 00000002:00100000:1.0:1788876621.094431:0:33363:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa061889b9c00 x1875771511113216/t68719477161(68719477161) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876637 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788876621.094442:0:33363:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788876621.094447:0:33363:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511113600/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:1.0:1788876621.094469:0:33363:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33363:1875771511113600:192.168.203.111@tcp:35:grep.0 00000100:00100000:1.0:1788876621.094548:0:33363:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.233136:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ecd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511116544:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876621.237797:0:2395:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa06189722300 00000020:00200000:1.0:1788876621.237814:0:2395:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876621.237833:0:2395:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa06189722300 00000100:00100000:1.0:1788876621.237982:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511116544:192.168.203.111@tcp:3:grep.0 00000080:00200000:1.0:1788876621.251089:0:33369:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.251123:0:33369:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.251145:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 224 lru_type: 1 00010000:00010000:1.0:1788876621.251163:0:33369:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 224 lru_type: 1 00000080:00200000:1.0:1788876621.251173:0:33369:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.251180:0:33369:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.251184:0:33369:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876621.251193:0:33369:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac2127bf8 00010000:00010000:1.0:1788876621.251197:0:33369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 224 lru_type: 1 00010000:00010000:1.0:1788876621.251207:0:33369:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 225 lru_type: 1 00000080:00200000:1.0:1788876621.251222:0:33369:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.251236:0:33369:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f5.replay-single-219, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa0618817db00, open_flags 8100, mode 0 opened 0 00000080:00200000:1.0:1788876621.251242:0:33369:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f5.replay-single-219, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:1.0:1788876621.251257:0:33369:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f5.replay-single-219,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:1.0:1788876621.251319:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33369 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.251328:0:33369:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33369 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:1.0:1788876621.251341:0:33369:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:1.0:1788876621.251352:0:33369:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898df80 x1875771511117056/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:1.0:1788876621.251362:0:33369:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33369 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.254481:0:33369:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898df80 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33369:1875771511117056:192.168.203.111@tcp:101:grep.0 00000100:00100000:1.0:1788876621.255101:0:33369:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.258234:0:33369:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898df80 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33369:1875771511117056:192.168.203.111@tcp:101:grep.0 00010000:00010000:1.0:1788876621.258262:0:33369:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258287:0:33369:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258295:0:33369:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258307:0:33369:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258318:0:33369:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258347:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258356:0:33369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258364:0:33369:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.258419:0:33369:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa0618898df80 x1875771511117056/t68719477173(68719477173) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876637 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:1.0:1788876621.258435:0:33369:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898df80 x1875771511117056/t68719477173(68719477173) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788876637 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.258453:0:33369:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:1.0:1788876621.258467:0:33369:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258479:0:33369:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/261 (0 0) 00000080:00200000:1.0:1788876621.258486:0:33369:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876621.258529:0:33369:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.258540:0:33369:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.258566:0:33369:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x105:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000008:00010000:1.0:1788876621.258630:0:33369:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 0 blocks 0 [cma]time: 0 0 0 by oinfo size 8 blocks 8 [cma]time 1788876564 1788876564 0 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.258698:0:33369:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x105:0x0](ffffa0619828db88): rc = 0 00000080:00200000:1.0:1788876621.258728:0:33369:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x105:0x0]: layout version change: 4294967294 -> 65535 00000080:00010000:1.0:1788876621.258748:0:33369:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x105:0x0](ffffa0619828db88) for lock 0x7d14dcdc37048227 00000080:00010000:1.0:1788876621.258759:0:33369:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x105:0x0](ffffa0619828db88) 00000080:00200008:1.0:1788876621.258773:0:33369:0:(file.c:1090:ll_file_open()) START file f5.replay-single-219:[0x200001b71:0x105:0x0](ffffa0619828db88), flags 100400 00000002:00100000:1.0:1788876621.258793:0:33369:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa0618898df80 x1875771511117056/t68719477173(68719477173) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876637 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:1.0:1788876621.258848:0:33369:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048227 from it ffffa0618908c240 00010000:00010000:1.0:1788876621.258853:0:33369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.258874:0:33369:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.258897:0:33369:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x105:0x0](ffffa0619828db88), inode mode 81a4 mask 44 00000080:00200008:1.0:1788876621.258918:0:33369:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-219:[0x200001b71:0x105:0x0](ffffa0619828db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.258925:0:33369:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x105:0x0](ffffa0619828db88),name=f5.replay-single-219 00000002:00010000:1.0:1788876621.258942:0:33369:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x105:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.258952:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.258965:0:33369:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.258984:0:33369:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.258989:0:33369:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x105:0x0](ffffa0619828db88) 00000080:00010000:1.0:1788876621.258992:0:33369:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x105:0x0](ffffa0619828db88) for lock 0x7d14dcdc37048227 00000080:00010000:1.0:1788876621.258996:0:33369:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048227 from it ffffb95ac2127cd8 00010000:00010000:1.0:1788876621.259000:0:33369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.259008:0:33369:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e400/0x7d14dcdc37048227 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x105:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abf23 expref: -99 pid: 33369 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.259037:0:33369:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876621.259045:0:33369:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x105:0x0] 00000008:00010000:1.0:1788876621.259058:0:33369:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.259061:0:33369:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2ab0, osc lock ffffa06186792850, flags 0x400001002 00010000:00010000:1.0:1788876621.259075:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.259086:0:33369:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788876621.259099:0:33369:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8, setting rss=8; kms=8, end=18446744073709551615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:1.0:1788876621.259109:0:33369:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.259113:0:33369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.259134:0:33369:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876621.259145:0:33369:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.259156:0:33369:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x105:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.259160:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a7780 (ffffa06188721488/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.259163:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a7780 (ffffa06188721488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.259166:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2ab0 (ffffa06188721098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.259170:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a7780 (ffffa06188721488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.259173:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2ab0 (ffffa06188721488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.259181:0:33369:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x105:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:1.0:1788876621.259199:0:33369:0:(file.c:2483:do_file_read_iter()) START file f5.replay-single-219:[0x200001b71:0x105:0x0], ppos: 0, count: 32768 00000080:00200000:1.0:1788876621.259218:0:33369:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:1.0:1788876621.259232:0:33369:0:(file.c:2090:ll_file_io_generic()) f5.replay-single-219: read ppos: 0, bytes: 32768 00000080:00200000:1.0:1788876621.259236:0:33369:0:(file.c:1848:ll_io_set_mirror()) f5.replay-single-219: desiginated mirror: 0 00000020:00200000:1.0:1788876621.259241:0:33369:0:(cl_io.c:214:cl_io_rw_init()) header@00000000075c5353[0x110, 1, [0x200001b71:0x105:0x0] hash] 00000020:00200000:1.0:1788876621.259243:0:33369:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 32768) 0 0 00000080:00200000:1.0:1788876621.259245:0:33369:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x105:0x0] ignore/verify layout 0/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.259256:0:33369:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 32768) 32768, 32768 00020000:00200000:1.0:1788876621.259259:0:33369:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.259269:0:33369:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:1.0:1788876621.259276:0:33369:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1788876621.259280:0:33369:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2ab0, osc lock ffffa06186792850, flags 0x0 00010000:00010000:1.0:1788876621.259289:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.259299:0:33369:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.259314:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.259322:0:33369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.259329:0:33369:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 5/1,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.259338:0:33369:0:(vvp_io.c:835:vvp_io_read_start()) f5.replay-single-219: read [0, 32768) 00000001:00010000:1.0:1788876621.259348:0:33369:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x105:0x0] 00000008:00010000:1.0:1788876621.259353:0:33369:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.259355:0:33369:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2cb0, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.259361:0:33369:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.259370:0:33369:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876621.259378:0:33369:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.259405:0:33369:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 7/1,1 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.259413:0:33369:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 6/1,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:1.0:1788876621.259432:0:33369:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.259438:0:33369:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x105:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.259440:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a7780 (ffffa06188721488/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.259443:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a7780 (ffffa06188721488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.259446:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2cb0 (ffffa06188721098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.259450:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a7780 (ffffa06188721488/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.259452:0:33369:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2cb0 (ffffa06188721488/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.259458:0:33369:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:0 00000008:00010000:1.0:1788876621.259493:0:33369:0:(osc_cache.c:3059:osc_queue_sync_pages()) ### no dlm lock: extent: ffffa061c20649c0 00000008:00010000:1.0:1788876621.259528:0:30612:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### no dlm lock: extent: ffffa061c20649c0 00010000:00010000:1.0:1788876621.259563:0:30612:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788876621.259602:0:30612:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618898d180 - object 0x2c0000401:559 offset 0<>4096 00000100:00100000:1.0:1788876621.259646:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788876621.259662:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511117184:192.168.203.111@tcp:3:grep.0 00000008:00010000:1.0:1788876621.263686:0:2395:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffffa061c20649c0 00000020:00200000:1.0:1788876621.263700:0:2395:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1788876621.263718:0:2395:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffffa061c20649c0 00000100:00100000:1.0:1788876621.263756:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511117184:192.168.203.111@tcp:3:grep.0 00000100:00100000:1.0:1788876621.268475:0:33369:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecca80 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:33369:1875771511117440:192.168.203.111@tcp:35:grep.0 00000002:00100000:1.0:1788876621.268517:0:33369:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa0618898df80 x1875771511117056/t68719477173(68719477173) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788876621.312670:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.312705:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.312726:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 229 lru_type: 1 00010000:00010000:1.0:1788876621.312819:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 229 lru_type: 1 00000080:00200000:1.0:1788876621.312829:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.312838:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.312841:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876621.312849:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.312854:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 229 lru_type: 1 00010000:00010000:1.0:1788876621.312934:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00000080:00200000:1.0:1788876621.312952:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.312960:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-1, flags=8 00000080:00200008:1.0:1788876621.312968:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-1:[0x200001b71:0x2b:0x0](ffffa06198225b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.312972:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x2b:0x0](ffffa06198225b88),name=f5.replay-single-1 00000002:00010000:1.0:1788876621.312985:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x2b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.312993:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ac000/0x7d14dcdc37047bf9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa075 expref: -99 pid: 33137 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.313003:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ac000/0x7d14dcdc37047bf9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa075 expref: -99 pid: 33137 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.313033:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.313037:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x2b:0x0](ffffa06198225b88) 00000080:00010000:1.0:1788876621.313040:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x2b:0x0](ffffa06198225b88) for lock 0x7d14dcdc37047bf9 00000080:00010000:1.0:1788876621.313044:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf9 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.313048:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ac000/0x7d14dcdc37047bf9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa075 expref: -99 pid: 33137 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.313055:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ac000/0x7d14dcdc37047bf9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa075 expref: -99 pid: 33137 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 1 00000080:00200000:1.0:1788876621.313075:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x2b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.313085:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x2b:0x0] 00000008:00010000:1.0:1788876621.313105:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.313108:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2cb0, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.313126:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ab2de840/0x7d14dcdc370469d8 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1c2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a54f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1788876621.313136:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ab2de840/0x7d14dcdc370469d8 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a54f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00000020:00200000:1.0:1788876621.313147:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.313151:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ab2de840/0x7d14dcdc370469d8 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a54f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1788876621.313159:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ab2de840/0x7d14dcdc370469d8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a54f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:1.0:1788876621.313180:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.313192:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x2b:0x0] updating i_size 6 i_blocks 8 00000020:00010000:1.0:1788876621.313197:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.313200:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.313203:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2cb0 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.313206:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.313208:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2cb0 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.313220:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x2b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.313593:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.313604:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.313613:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00010000:00010000:1.0:1788876621.313622:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00000080:00200000:1.0:1788876621.313629:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.313634:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.313637:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876621.313641:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.313644:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 230 lru_type: 1 00010000:00010000:1.0:1788876621.313651:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00000080:00200000:1.0:1788876621.313671:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.313679:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-1, flags=0 00000080:00200000:1.0:1788876621.313683:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.313690:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.313697:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00010000:00010000:1.0:1788876621.313705:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00000080:00200000:1.0:1788876621.313712:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.313716:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.313719:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37047bf2 00000080:00010000:1.0:1788876621.313723:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047bf2 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.313726:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 231 lru_type: 1 00010000:00010000:1.0:1788876621.313811:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00000080:00200000:1.0:1788876621.313821:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.313827:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-1, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.313852:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00010000:00010000:1.0:1788876621.314078:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ac000/0x7d14dcdc37047bf9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa075 expref: -99 pid: 33137 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 1 00000080:00200000:1.0:1788876621.314106:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x2b:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.314143:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ac000/0x7d14dcdc37047bf9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa075 expref: -99 pid: 33137 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 1 00010000:00010000:1.0:1788876621.314152:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00010000:00010000:1.0:1788876621.314158:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.314161:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ac000/0x7d14dcdc37047bf9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x2b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa075 expref: -99 pid: 33137 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 1 00010000:00010000:1.0:1788876621.314179:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934a00/0x7d14dcdc37047bf2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313a9f5d expref: -99 pid: 32651 timeout: 0 lvb_type: 0 lru_score: 232 lru_type: 1 00000020:00100000:1.0:1788876621.314196:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.314209:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898df80 x1875771511118464/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.314231:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898df80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511118464:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.314360:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00010000:00010000:1.0:1788876621.422764:0:33151:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935800/0x7d14dcdc37047c85 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa2f9 expref: -99 pid: 33161 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.422775:0:33151:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935800/0x7d14dcdc37047c85 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa2f9 expref: -99 pid: 33161 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00010000:1.0:1788876621.422781:0:33151:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047c85 from it ffffb95ac0ddbd78 00010000:00010000:1.0:1788876621.422797:0:33151:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935800/0x7d14dcdc37047c85 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa2f9 expref: -99 pid: 33161 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.422804:0:33151:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935800/0x7d14dcdc37047c85 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x37:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa2f9 expref: -99 pid: 33161 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.427787:0:33151:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x34:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.427807:0:33151:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.427839:0:33151:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x34:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.427900:0:33151:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x34:0x0](ffffa06198178908) 00000008:00010000:1.0:1788876621.427936:0:33151:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876613 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000006879953d/0x7d14dcdc37046a95 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1c6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a56ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1788876621.427988:0:33151:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x35:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788876621.427994:0:33151:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.428004:0:33151:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x35:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.428034:0:33151:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x35:0x0](ffffa06198226c08) 00000008:00010000:1.0:1788876621.428051:0:33151:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876613 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000002e5e89d/0x7d14dcdc37046aaa lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1c7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5726 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000100:00100000:1.0:1788876621.517002:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188949180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511126656:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.517040:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517054:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517085:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517098:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517113:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517122:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517129:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.517142:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188949180 x1875771511126656/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.517156:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517170:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.517176:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.517201:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.517207:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482c8 00000080:00010000:1.0:1788876621.517212:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.517216:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482c8 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.517225:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.517236:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.517262:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.517273:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-119, flags=8 00000080:00200008:1.0:1788876621.517284:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-119:[0x200001b71:0xa1:0x0](ffffa06198253a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.517289:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xa1:0x0](ffffa06198253a88),name=f5.replay-single-119 00000002:00010000:1.0:1788876621.517312:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xa1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.517324:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ad000/0x7d14dcdc37047f6b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab177 expref: -99 pid: 33268 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.517337:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ad000/0x7d14dcdc37047f6b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab177 expref: -99 pid: 33268 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.517344:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.517348:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xa1:0x0](ffffa06198253a88) 00000080:00010000:1.0:1788876621.517351:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xa1:0x0](ffffa06198253a88) for lock 0x7d14dcdc37047f6b 00000080:00010000:1.0:1788876621.517355:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047f6b from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.517358:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ad000/0x7d14dcdc37047f6b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xa1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab177 expref: -99 pid: 33268 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.517366:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ad000/0x7d14dcdc37047f6b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab177 expref: -99 pid: 33268 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.517538:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.517557:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xa1:0x0] 00000008:00010000:1.0:1788876621.517581:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.517585:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2cb0, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.517606:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000097072fa7/0x7d14dcdc37047386 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6ec6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.517621:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000097072fa7/0x7d14dcdc37047386 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6ec6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.517635:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.517639:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000097072fa7/0x7d14dcdc37047386 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6ec6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.517647:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000097072fa7/0x7d14dcdc37047386 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6ec6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.517660:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.517672:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xa1:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.517676:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.517680:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.517683:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2cb0 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.517688:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.517691:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2cb0 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.517703:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa1:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.517737:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.517745:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.517752:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.517767:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.517774:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.517778:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.517781:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482c8 00000080:00010000:1.0:1788876621.517785:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482c8 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.517788:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.517796:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.517805:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.517813:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-119, flags=0 00000080:00200000:1.0:1788876621.517817:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.517823:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.517829:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.517837:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.517843:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.517847:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.517849:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482c8 00000080:00010000:1.0:1788876621.517852:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482c8 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.517861:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.517869:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.517878:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.517884:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-119, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.517911:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.517981:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ad000/0x7d14dcdc37047f6b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab177 expref: -99 pid: 33268 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.517993:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xa1:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.518040:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ad000/0x7d14dcdc37047f6b lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xa1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab177 expref: -99 pid: 33268 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.518050:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.518057:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.518060:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ad000/0x7d14dcdc37047f6b lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xa1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab177 expref: -99 pid: 33268 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.518080:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc370482c8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac2e9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.518098:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.518114:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898d180 x1875771511126912/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.518136:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898d180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511126912:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.518216:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.523827:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898d180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511126912:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.523899:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa1:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.523914:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.523940:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.523984:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xa1:0x0](ffffa06198253a88) 00000008:00010000:1.0:1788876621.524030:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 8 blocks 8 [cma]time 1788876561 1788876561 1788876618 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000097072fa7/0x7d14dcdc37047386 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6ec6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.524119:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.524141:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.524206:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370482cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.524217:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370482cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.524227:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370482cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.524247:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898ea00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511127040:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.524311:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00010000:00010000:1.0:1788876621.565998:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876621.648085:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190e30e00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511128832:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.648185:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa5:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788876621.648199:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.648226:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa5:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.648278:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xa5:0x0](ffffa0619825cb08) 00000008:00010000:1.0:1788876621.648320:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 8 blocks 8 [cma]time 1788876561 1788876561 1788876618 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003f900b45/0x7d14dcdc370473da lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6fa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.649835:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.649872:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.649944:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.649955:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.649964:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.649990:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511128960:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.650128:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.662222:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898f100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511128960:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.662255:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662269:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662307:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662319:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662333:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662342:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662350:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.662361:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898f100 x1875771511128960/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.662376:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662408:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.662415:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.662445:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.662451:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482f2 00000080:00010000:1.0:1788876621.662456:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.662460:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482f2 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.662470:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.662479:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.662506:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.662519:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-124, flags=8 00000080:00200008:1.0:1788876621.662534:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-124:[0x200001b71:0xa6:0x0](ffffa0619823f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.662540:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xa6:0x0](ffffa0619823f448),name=f5.replay-single-124 00000002:00010000:1.0:1788876621.662563:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xa6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.662575:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994000/0x7d14dcdc37047f8e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab226 expref: -99 pid: 33273 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.662587:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994000/0x7d14dcdc37047f8e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab226 expref: -99 pid: 33273 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.662594:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.662598:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xa6:0x0](ffffa0619823f448) 00000080:00010000:1.0:1788876621.662601:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xa6:0x0](ffffa0619823f448) for lock 0x7d14dcdc37047f8e 00000080:00010000:1.0:1788876621.662606:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047f8e from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.662609:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994000/0x7d14dcdc37047f8e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xa6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab226 expref: -99 pid: 33273 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.662616:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994000/0x7d14dcdc37047f8e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab226 expref: -99 pid: 33273 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.662637:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa6:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.662658:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xa6:0x0] 00000008:00010000:1.0:1788876621.662685:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.662690:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.662714:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d83218ed/0x7d14dcdc370473ef lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6fde expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.662726:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d83218ed/0x7d14dcdc370473ef lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6fde expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.662741:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.662744:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d83218ed/0x7d14dcdc370473ef lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6fde expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.662752:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d83218ed/0x7d14dcdc370473ef lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6fde expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.662762:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.662778:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xa6:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.662782:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.662785:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.662788:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.662792:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.662795:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.662809:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa6:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.662845:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.662853:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.662860:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.662874:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.662882:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.662886:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.662888:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482f2 00000080:00010000:1.0:1788876621.662893:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482f2 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.662895:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.662903:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.662911:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.662919:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-124, flags=0 00000080:00200000:1.0:1788876621.662923:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.662929:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.662935:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.662943:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.662949:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.662952:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.662955:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482f2 00000080:00010000:1.0:1788876621.662958:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482f2 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.662967:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.662974:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.662983:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.662988:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-124, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.663012:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.663082:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994000/0x7d14dcdc37047f8e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab226 expref: -99 pid: 33273 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.663094:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xa6:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.663129:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994000/0x7d14dcdc37047f8e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xa6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab226 expref: -99 pid: 33273 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.663138:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.663145:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.663148:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188994000/0x7d14dcdc37047f8e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xa6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab226 expref: -99 pid: 33273 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.663167:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac3e5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.663184:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.663199:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898f100 x1875771511129216/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.663220:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129216:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.663295:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.672951:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898f100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129216:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.673030:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa6:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.673046:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.673081:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.673140:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xa6:0x0](ffffa0619823f448) 00000008:00010000:1.0:1788876621.673181:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 8 blocks 8 [cma]time 1788876561 1788876561 1788876618 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d83218ed/0x7d14dcdc370473ef lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1ff:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6fde expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.673264:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.673288:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.673356:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.673365:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.673375:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.673462:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129344:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.673534:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.683833:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898f800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129344:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.683881:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.683893:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.683906:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.683916:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.683928:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.683935:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.683942:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.683953:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898f800 x1875771511129344/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.683966:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.683981:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.683987:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.684029:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.684036:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482f9 00000080:00010000:1.0:1788876621.684040:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.684043:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482f9 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.684046:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.684056:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.684081:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.684092:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-125, flags=8 00000080:00200008:1.0:1788876621.684102:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-125:[0x200001b71:0xa7:0x0](ffffa0619823b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.684108:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xa7:0x0](ffffa0619823b248),name=f5.replay-single-125 00000002:00010000:1.0:1788876621.684129:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xa7:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.684140:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516400/0x7d14dcdc37047f95 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab249 expref: -99 pid: 33274 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.684152:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516400/0x7d14dcdc37047f95 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab249 expref: -99 pid: 33274 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.684159:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.684164:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xa7:0x0](ffffa0619823b248) 00000080:00010000:1.0:1788876621.684166:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xa7:0x0](ffffa0619823b248) for lock 0x7d14dcdc37047f95 00000080:00010000:1.0:1788876621.684171:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047f95 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.684174:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516400/0x7d14dcdc37047f95 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xa7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab249 expref: -99 pid: 33274 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.684192:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516400/0x7d14dcdc37047f95 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab249 expref: -99 pid: 33274 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.684213:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.684224:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xa7:0x0] 00000008:00010000:1.0:1788876621.684243:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.684246:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.684263:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.684274:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.684287:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.684290:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.684297:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.684307:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.684317:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xa7:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.684321:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.684324:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.684327:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.684331:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.684339:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.684350:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa7:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.684406:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.684415:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.684422:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.684432:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.684439:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.684443:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.684446:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482f9 00000080:00010000:1.0:1788876621.684450:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482f9 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.684453:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.684461:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.684470:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.684477:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-125, flags=0 00000080:00200000:1.0:1788876621.684481:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.684487:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.684494:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.684502:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.684515:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.684519:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.684522:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370482f9 00000080:00010000:1.0:1788876621.684526:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370482f9 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.684529:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.684536:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.684545:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.684551:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-125, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.684574:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.684621:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516400/0x7d14dcdc37047f95 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab249 expref: -99 pid: 33274 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.684633:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xa7:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.684664:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516400/0x7d14dcdc37047f95 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xa7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab249 expref: -99 pid: 33274 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.684673:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.684680:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.684683:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516400/0x7d14dcdc37047f95 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xa7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab249 expref: -99 pid: 33274 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.684707:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc370482f9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac40f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.684725:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.684733:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898f800 x1875771511129600/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.684754:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129600:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.684878:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.688362:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898f800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129600:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.688454:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa7:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.688469:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.688492:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.688537:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xa7:0x0](ffffa0619823b248) 00000008:00010000:1.0:1788876621.688570:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 8 blocks 8 [cma]time 1788876561 1788876561 1788876618 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.688646:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.688665:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.688720:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.688744:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.688755:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.688786:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898ed80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129728:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.688840:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.691843:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898ed80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511129728:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.691934:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.691948:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.691962:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.691976:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.691990:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.691999:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.692008:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.692030:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898ed80 x1875771511129728/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.692065:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.692080:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.692087:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.692112:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.692119:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048300 00000080:00010000:1.0:1788876621.692124:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.692128:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048300 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.692131:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.692825:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.692973:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.692986:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-126, flags=8 00000080:00200008:1.0:1788876621.692997:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-126:[0x200001b71:0xa8:0x0](ffffa0619823ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.693004:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xa8:0x0](ffffa0619823ec08),name=f5.replay-single-126 00000002:00010000:1.0:1788876621.693038:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xa8:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.693053:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516600/0x7d14dcdc37047f9c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab26c expref: -99 pid: 33275 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.693066:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516600/0x7d14dcdc37047f9c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xa8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab26c expref: -99 pid: 33275 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.693074:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.693089:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xa8:0x0](ffffa0619823ec08) 00000080:00010000:1.0:1788876621.693092:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xa8:0x0](ffffa0619823ec08) for lock 0x7d14dcdc37047f9c 00000080:00010000:1.0:1788876621.693097:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047f9c from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.693101:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516600/0x7d14dcdc37047f9c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xa8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab26c expref: -99 pid: 33275 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.693109:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516600/0x7d14dcdc37047f9c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab26c expref: -99 pid: 33275 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.693130:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xa8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.693141:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xa8:0x0] 00000008:00010000:1.0:1788876621.693161:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.693165:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.693182:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cff0eb23/0x7d14dcdc37047419 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x200:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a704e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.693195:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cff0eb23/0x7d14dcdc37047419 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a704e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.693208:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.693212:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cff0eb23/0x7d14dcdc37047419 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a704e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.693221:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cff0eb23/0x7d14dcdc37047419 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a704e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.693231:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.693242:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xa8:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.693251:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.693255:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.693258:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.693263:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.693266:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.693276:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xa8:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.693307:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.693315:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.693324:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.693332:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.693340:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.693344:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.693347:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048300 00000080:00010000:1.0:1788876621.693351:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048300 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.693354:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.693363:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.693372:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.695026:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-126, flags=0 00000080:00200000:1.0:1788876621.695035:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.695055:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.695084:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.695100:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.695109:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.695117:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.695120:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048300 00000080:00010000:1.0:1788876621.695127:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048300 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.695131:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.695143:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.695158:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.695164:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-126, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.695197:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.695255:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516600/0x7d14dcdc37047f9c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xa8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab26c expref: -99 pid: 33275 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.695267:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xa8:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.695309:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516600/0x7d14dcdc37047f9c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xa8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab26c expref: -99 pid: 33275 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.695318:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.695342:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.695346:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516600/0x7d14dcdc37047f9c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xa8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab26c expref: -99 pid: 33275 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.695365:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048300 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac439 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.698994:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.699021:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898ed80 x1875771511129984/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.799665:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190e31c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511132416:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.799691:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799703:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799715:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799737:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799751:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799779:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799787:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.799799:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06190e31c00 x1875771511132416/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.799813:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799826:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.799832:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.799867:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.799873:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048331 00000080:00010000:1.0:1788876621.799879:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.799882:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048331 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.799885:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.799895:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.799918:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.799928:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-132, flags=8 00000080:00200008:1.0:1788876621.799938:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-132:[0x200001b71:0xae:0x0](ffffa06198261148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.799948:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xae:0x0](ffffa06198261148),name=f5.replay-single-132 00000002:00010000:1.0:1788876621.799968:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xae:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.799978:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2800/0x7d14dcdc37047fc6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xae:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab33e expref: -99 pid: 33281 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.799989:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2800/0x7d14dcdc37047fc6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab33e expref: -99 pid: 33281 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.799995:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.800000:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xae:0x0](ffffa06198261148) 00000080:00010000:1.0:1788876621.800003:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xae:0x0](ffffa06198261148) for lock 0x7d14dcdc37047fc6 00000080:00010000:1.0:1788876621.800007:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047fc6 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.800010:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2800/0x7d14dcdc37047fc6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab33e expref: -99 pid: 33281 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.800029:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2800/0x7d14dcdc37047fc6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab33e expref: -99 pid: 33281 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.800048:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xae:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876621.800058:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xae:0x0] 00000008:00010000:1.0:1788876621.800077:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.800079:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb24b0, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.800096:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000006c61f8c8/0x7d14dcdc37047497 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x203:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a719e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.800107:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000006c61f8c8/0x7d14dcdc37047497 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a719e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.800129:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.800133:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000006c61f8c8/0x7d14dcdc37047497 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a719e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.800139:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000006c61f8c8/0x7d14dcdc37047497 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a719e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.800148:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.800159:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xae:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.800162:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.800165:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.800167:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb24b0 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.800170:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.800172:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb24b0 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.800182:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xae:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.800214:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.800222:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.800229:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.800236:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.800241:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.800245:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.800248:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048331 00000080:00010000:1.0:1788876621.800255:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048331 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.800258:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.800265:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.800273:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.800280:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-132, flags=0 00000080:00200000:1.0:1788876621.800284:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.800290:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.800296:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.800303:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.800309:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.800313:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.800315:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048331 00000080:00010000:1.0:1788876621.800318:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048331 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.800321:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.800327:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.800333:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.800342:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-132, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.800361:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.800445:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2800/0x7d14dcdc37047fc6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xae:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab33e expref: -99 pid: 33281 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.800456:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xae:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.800490:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2800/0x7d14dcdc37047fc6 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab33e expref: -99 pid: 33281 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.800498:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.800504:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.800507:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2800/0x7d14dcdc37047fc6 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xae:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab33e expref: -99 pid: 33281 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.800525:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048331 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac55f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.800540:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.800547:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898d500 x1875771511132672/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.800567:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898d500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511132672:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.800630:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.804270:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898d500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511132672:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.804328:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xae:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.804358:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.804466:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xae:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.804512:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xae:0x0](ffffa06198261148) 00000008:00010000:1.0:1788876621.804547:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876618 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000006c61f8c8/0x7d14dcdc37047497 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x203:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a719e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.804623:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.804644:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.804707:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.804716:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.804727:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.804748:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511132800:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.804876:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.807723:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511132800:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.807835:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.807848:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.807943:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.807974:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.807989:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.807997:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.808005:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.808027:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898c000 x1875771511132800/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.808042:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.808057:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.808064:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.808090:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.808097:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048338 00000080:00010000:1.0:1788876621.808102:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.808105:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048338 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.808108:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.808130:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.808157:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.808168:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-133, flags=8 00000080:00200008:1.0:1788876621.808179:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-133:[0x200001b71:0xaf:0x0](ffffa061981bf448), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.808184:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xaf:0x0](ffffa061981bf448),name=f5.replay-single-133 00000002:00010000:1.0:1788876621.808202:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xaf:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.808213:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2e00/0x7d14dcdc37047fcd lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xaf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab361 expref: -99 pid: 33282 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.808224:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2e00/0x7d14dcdc37047fcd lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xaf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab361 expref: -99 pid: 33282 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.808231:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.808235:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xaf:0x0](ffffa061981bf448) 00000080:00010000:1.0:1788876621.808238:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xaf:0x0](ffffa061981bf448) for lock 0x7d14dcdc37047fcd 00000080:00010000:1.0:1788876621.808242:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047fcd from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.808245:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2e00/0x7d14dcdc37047fcd lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xaf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab361 expref: -99 pid: 33282 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.808253:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2e00/0x7d14dcdc37047fcd lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xaf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab361 expref: -99 pid: 33282 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.808272:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xaf:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.808281:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xaf:0x0] 00000008:00010000:1.0:1788876621.808306:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.808309:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.808327:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068119bad/0x7d14dcdc370474ac lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x204:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a71d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.808338:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068119bad/0x7d14dcdc370474ac lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a71d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.808350:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.808354:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068119bad/0x7d14dcdc370474ac lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a71d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.808362:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068119bad/0x7d14dcdc370474ac lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a71d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.808545:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.808559:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xaf:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.808563:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.808566:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.808569:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.808573:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.808576:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.808588:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xaf:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.808617:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.808625:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.808633:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.808650:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.808657:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.808661:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.808663:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048338 00000080:00010000:1.0:1788876621.808668:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048338 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.808671:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.808678:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.808687:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.808694:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-133, flags=0 00000080:00200000:1.0:1788876621.808697:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.808703:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.808709:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.808717:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.808724:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.808794:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.808796:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048338 00000080:00010000:1.0:1788876621.808801:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048338 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.808809:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.808817:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.808826:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.808831:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-133, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.808923:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.808974:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2e00/0x7d14dcdc37047fcd lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xaf:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab361 expref: -99 pid: 33282 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.808985:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xaf:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.809028:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2e00/0x7d14dcdc37047fcd lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xaf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab361 expref: -99 pid: 33282 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.809036:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.809042:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.809045:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a2e00/0x7d14dcdc37047fcd lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xaf:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab361 expref: -99 pid: 33282 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.809064:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048338 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac589 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.809081:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.809096:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898c000 x1875771511133056/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.809117:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133056:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.809248:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.812697:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133056:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.812824:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xaf:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.812836:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.812926:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xaf:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.812972:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xaf:0x0](ffffa061981bf448) 00000008:00010000:1.0:1788876621.813007:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876618 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068119bad/0x7d14dcdc370474ac lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x204:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a71d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.813097:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.813117:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.813177:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.813186:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.813196:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.813216:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898fb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133184:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.813323:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.816593:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898fb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133184:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.816636:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816649:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816663:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816674:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816688:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816696:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816704:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.816715:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898fb80 x1875771511133184/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.816834:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816851:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.816865:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.816898:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.816905:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704833f 00000080:00010000:1.0:1788876621.816910:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.816914:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704833f from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.816917:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.816927:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.816953:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.816965:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-134, flags=8 00000080:00200008:1.0:1788876621.816977:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-134:[0x200001b71:0xb0:0x0](ffffa061981bcb08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.816983:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xb0:0x0](ffffa061981bcb08),name=f5.replay-single-134 00000002:00010000:1.0:1788876621.817006:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xb0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.817030:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516000/0x7d14dcdc37047fd4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab384 expref: -99 pid: 33283 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.817043:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516000/0x7d14dcdc37047fd4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab384 expref: -99 pid: 33283 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.817050:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.817055:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xb0:0x0](ffffa061981bcb08) 00000080:00010000:1.0:1788876621.817058:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xb0:0x0](ffffa061981bcb08) for lock 0x7d14dcdc37047fd4 00000080:00010000:1.0:1788876621.817061:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047fd4 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.817064:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516000/0x7d14dcdc37047fd4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xb0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab384 expref: -99 pid: 33283 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.817084:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516000/0x7d14dcdc37047fd4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab384 expref: -99 pid: 33283 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.817106:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb0:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.817118:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xb0:0x0] 00000008:00010000:1.0:1788876621.817143:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.817146:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.817164:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e35a8b1a/0x7d14dcdc370474c1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x204:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a720e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.817177:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e35a8b1a/0x7d14dcdc370474c1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a720e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.817190:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.817194:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e35a8b1a/0x7d14dcdc370474c1 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a720e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.817202:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e35a8b1a/0x7d14dcdc370474c1 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a720e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.817213:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.817225:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xb0:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.817229:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.817232:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.817235:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.817239:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.817248:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.817261:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb0:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.817297:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.817306:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.817313:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.817322:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.817328:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.817332:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.817335:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704833f 00000080:00010000:1.0:1788876621.817338:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704833f from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.817341:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.817349:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.817357:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.817366:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-134, flags=0 00000080:00200000:1.0:1788876621.817371:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.817644:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.817652:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.817661:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.817675:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.817679:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.817682:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704833f 00000080:00010000:1.0:1788876621.817686:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704833f from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.817689:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.817696:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.817705:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.817711:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-134, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.817810:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.817922:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516000/0x7d14dcdc37047fd4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb0:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab384 expref: -99 pid: 33283 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.817936:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xb0:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.817978:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516000/0x7d14dcdc37047fd4 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xb0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab384 expref: -99 pid: 33283 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.817986:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.817991:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.817994:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181516000/0x7d14dcdc37047fd4 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xb0:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab384 expref: -99 pid: 33283 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.818030:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704833f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac5b3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.818050:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.818060:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898fb80 x1875771511133440/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.818085:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898fb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133440:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.818225:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.823919:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898fb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133440:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.823986:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb0:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.824000:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.824040:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb0:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.824095:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xb0:0x0](ffffa061981bcb08) 00000008:00010000:1.0:1788876621.824131:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876618 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e35a8b1a/0x7d14dcdc370474c1 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x204:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a720e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.824213:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.824236:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.824301:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.824310:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.824319:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.824351:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898e300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133568:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.824559:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.831324:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898e300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133568:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.831351:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831363:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831645:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831660:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831676:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831685:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831694:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.831706:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898e300 x1875771511133568/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.831802:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831819:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.831826:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.831911:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.831919:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048346 00000080:00010000:1.0:1788876621.831924:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.831928:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048346 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.831931:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.831942:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.831966:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.831977:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-135, flags=8 00000080:00200008:1.0:1788876621.831988:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-135:[0x200001b71:0xb1:0x0](ffffa061981bc2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.831994:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xb1:0x0](ffffa061981bc2c8),name=f5.replay-single-135 00000002:00010000:1.0:1788876621.832024:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xb1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.832035:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baa200/0x7d14dcdc37047fdb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3a7 expref: -99 pid: 33284 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.832048:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baa200/0x7d14dcdc37047fdb lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3a7 expref: -99 pid: 33284 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.832055:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.832065:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xb1:0x0](ffffa061981bc2c8) 00000080:00010000:1.0:1788876621.832069:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xb1:0x0](ffffa061981bc2c8) for lock 0x7d14dcdc37047fdb 00000080:00010000:1.0:1788876621.832073:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047fdb from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.832077:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baa200/0x7d14dcdc37047fdb lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xb1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab3a7 expref: -99 pid: 33284 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.832084:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baa200/0x7d14dcdc37047fdb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3a7 expref: -99 pid: 33284 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.832103:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb1:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.832113:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xb1:0x0] 00000008:00010000:1.0:1788876621.832132:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.832136:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.832152:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d7a262a9/0x7d14dcdc370474d6 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x205:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7246 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.832164:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d7a262a9/0x7d14dcdc370474d6 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7246 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.832177:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.832181:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d7a262a9/0x7d14dcdc370474d6 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7246 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.832188:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d7a262a9/0x7d14dcdc370474d6 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7246 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.832198:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.832209:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xb1:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.832221:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.832224:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.832228:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.832232:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.832234:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.832246:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb1:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.832277:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.832284:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.832291:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.832300:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.832306:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.832310:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.832313:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048346 00000080:00010000:1.0:1788876621.832317:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048346 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.832320:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.832327:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.832335:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.832342:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-135, flags=0 00000080:00200000:1.0:1788876621.832346:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.832352:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.832363:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.832372:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.832553:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.832558:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.832561:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048346 00000080:00010000:1.0:1788876621.832565:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048346 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.832568:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.832575:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.832647:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.832653:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-135, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.832677:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.832792:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baa200/0x7d14dcdc37047fdb lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb1:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab3a7 expref: -99 pid: 33284 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.832808:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xb1:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.832950:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baa200/0x7d14dcdc37047fdb lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xb1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab3a7 expref: -99 pid: 33284 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.832959:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.832973:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.832976:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baa200/0x7d14dcdc37047fdb lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xb1:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab3a7 expref: -99 pid: 33284 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.832996:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048346 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac5dd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.833022:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.833031:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898e300 x1875771511133824/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.833054:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898e300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133824:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.833126:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.840800:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898e300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133824:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.840858:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb1:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.840872:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.840899:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb1:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.840946:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xb1:0x0](ffffa061981bc2c8) 00000008:00010000:1.0:1788876621.840980:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876618 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d7a262a9/0x7d14dcdc370474d6 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x205:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7246 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.841068:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.841087:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.841294:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.841318:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.841329:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.841350:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133952:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.841456:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.843421:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898f480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511133952:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.843445:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843459:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843472:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843483:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843496:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843504:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843526:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.843538:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898f480 x1875771511133952/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.843551:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843563:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.843570:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.843594:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.843600:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704834d 00000080:00010000:1.0:1788876621.843605:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.843608:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704834d from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.843611:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.843622:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.843646:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.843658:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-136, flags=8 00000080:00200008:1.0:1788876621.843670:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-136:[0x200001b71:0xb2:0x0](ffffa0619825ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.843675:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xb2:0x0](ffffa0619825ba88),name=f5.replay-single-136 00000002:00010000:1.0:1788876621.843693:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xb2:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.843704:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886eda00/0x7d14dcdc37047fe2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3ca expref: -99 pid: 33285 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.843720:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886eda00/0x7d14dcdc37047fe2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3ca expref: -99 pid: 33285 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.843738:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.843743:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xb2:0x0](ffffa0619825ba88) 00000080:00010000:1.0:1788876621.843746:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xb2:0x0](ffffa0619825ba88) for lock 0x7d14dcdc37047fe2 00000080:00010000:1.0:1788876621.843751:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047fe2 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.843753:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886eda00/0x7d14dcdc37047fe2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xb2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab3ca expref: -99 pid: 33285 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.843760:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886eda00/0x7d14dcdc37047fe2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3ca expref: -99 pid: 33285 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.843779:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb2:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.843788:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xb2:0x0] 00000008:00010000:1.0:1788876621.843806:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.843809:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.843827:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc370474eb lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x205:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a727e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.843839:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc370474eb lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a727e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.843862:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.843867:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc370474eb lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a727e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.843885:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc370474eb lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a727e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.843896:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.843907:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xb2:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.843911:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.843914:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.843917:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.843921:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.843923:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.843934:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb2:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.843959:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.843967:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.843974:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.844089:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.844097:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.844102:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.844105:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704834d 00000080:00010000:1.0:1788876621.844109:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704834d from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.844113:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.844133:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.844148:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.844155:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-136, flags=0 00000080:00200000:1.0:1788876621.844159:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.844167:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.844174:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.844182:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.844188:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.844192:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.844194:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704834d 00000080:00010000:1.0:1788876621.844197:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704834d from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.844200:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.844206:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.844214:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.844220:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-136, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.844243:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.844295:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886eda00/0x7d14dcdc37047fe2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb2:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab3ca expref: -99 pid: 33285 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.844307:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xb2:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.844335:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886eda00/0x7d14dcdc37047fe2 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xb2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab3ca expref: -99 pid: 33285 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.844342:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.844348:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.844351:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886eda00/0x7d14dcdc37047fe2 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xb2:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab3ca expref: -99 pid: 33285 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.844367:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e200/0x7d14dcdc3704834d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac607 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.844406:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.844414:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898f480 x1875771511134208/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.844434:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898f480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511134208:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.844487:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.848057:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898f480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511134208:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.848121:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb2:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.848139:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.848168:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb2:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.848235:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xb2:0x0](ffffa0619825ba88) 00000008:00010000:1.0:1788876621.848272:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876618 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc370474eb lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x205:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a727e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876621.848351:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.848371:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.848457:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.848467:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.848477:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876621.848496:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511134336:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876621.848555:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.856263:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511134336:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876621.856290:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856302:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856314:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856325:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856351:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856359:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856366:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876621.856377:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898c700 x1875771511134336/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876637 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876621.856436:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856451:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876621.856458:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876621.856485:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.856491:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048354 00000080:00010000:1.0:1788876621.856496:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.856500:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048354 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.856504:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.856513:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.856550:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.856561:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-137, flags=8 00000080:00200008:1.0:1788876621.856573:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-137:[0x200001b71:0xb3:0x0](ffffa061982580c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.856580:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xb3:0x0](ffffa061982580c8),name=f5.replay-single-137 00000002:00010000:1.0:1788876621.856603:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xb3:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.856616:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baaa00/0x7d14dcdc37047fe9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3ed expref: -99 pid: 33286 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.856629:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baaa00/0x7d14dcdc37047fe9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xb3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3ed expref: -99 pid: 33286 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.856637:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.856642:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xb3:0x0](ffffa061982580c8) 00000080:00010000:1.0:1788876621.856645:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xb3:0x0](ffffa061982580c8) for lock 0x7d14dcdc37047fe9 00000080:00010000:1.0:1788876621.856649:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047fe9 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.856652:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baaa00/0x7d14dcdc37047fe9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xb3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab3ed expref: -99 pid: 33286 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.856660:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baaa00/0x7d14dcdc37047fe9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab3ed expref: -99 pid: 33286 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.856682:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb3:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.856692:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xb3:0x0] 00000008:00010000:1.0:1788876621.856713:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.856716:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.856749:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000023784b25/0x7d14dcdc37047500 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x206:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a72b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.856768:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000023784b25/0x7d14dcdc37047500 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a72b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.856781:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.856784:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000023784b25/0x7d14dcdc37047500 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a72b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.856793:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000023784b25/0x7d14dcdc37047500 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a72b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.856804:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.856815:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xb3:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.856818:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.856821:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.856824:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.856828:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.856831:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.856842:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb3:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.856885:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.856893:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.856901:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.856909:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.856925:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.856930:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.856932:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048354 00000080:00010000:1.0:1788876621.856936:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048354 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.856939:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.856946:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.856955:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.856962:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-137, flags=0 00000080:00200000:1.0:1788876621.856966:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.856971:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.856977:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.856984:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.856990:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.856994:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.856996:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048354 00000080:00010000:1.0:1788876621.856999:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048354 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.857002:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.857033:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.857042:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.857048:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-137, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.857072:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.857121:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baaa00/0x7d14dcdc37047fe9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xb3:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab3ed expref: -99 pid: 33286 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.857132:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xb3:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.857163:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baaa00/0x7d14dcdc37047fe9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xb3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab3ed expref: -99 pid: 33286 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.857171:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.857177:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.857180:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baaa00/0x7d14dcdc37047fe9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xb3:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab3ed expref: -99 pid: 33286 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.857198:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc37048354 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac631 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.857215:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.857223:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898c700 x1875771511134592/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.857250:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511134592:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.857312:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876621.865263:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511134592:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876621.865317:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xb3:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876621.865329:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876621.865356:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xb3:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.947717:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876621.947785:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704838c 00000080:00010000:1.0:1788876621.947793:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.947796:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704838c from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876621.947799:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876621.947809:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.947835:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.947847:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-144, flags=8 00000080:00200008:1.0:1788876621.947916:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-144:[0x200001b71:0xba:0x0](ffffa061982621c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876621.947923:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xba:0x0](ffffa061982621c8),name=f5.replay-single-144 00000002:00010000:1.0:1788876621.947945:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xba:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.947955:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a3400/0x7d14dcdc3704801a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab4e2 expref: -99 pid: 33293 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.947967:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a3400/0x7d14dcdc3704801a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab4e2 expref: -99 pid: 33293 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.947987:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.947992:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xba:0x0](ffffa061982621c8) 00000080:00010000:1.0:1788876621.947994:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xba:0x0](ffffa061982621c8) for lock 0x7d14dcdc3704801a 00000080:00010000:1.0:1788876621.947998:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704801a from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876621.948002:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a3400/0x7d14dcdc3704801a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab4e2 expref: -99 pid: 33293 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.948009:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a3400/0x7d14dcdc3704801a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab4e2 expref: -99 pid: 33293 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.948040:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xba:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876621.948050:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xba:0x0] 00000008:00010000:1.0:1788876621.948069:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876621.948073:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb24b0, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876621.948089:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000a378b3f7/0x7d14dcdc37047593 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x209:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a743e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.948100:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000a378b3f7/0x7d14dcdc37047593 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a743e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876621.948113:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876621.948117:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000a378b3f7/0x7d14dcdc37047593 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a743e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876621.948125:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000a378b3f7/0x7d14dcdc37047593 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x209:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a743e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876621.948145:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876621.948155:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xba:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876621.948159:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876621.948162:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.948165:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb24b0 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876621.948169:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876621.948172:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb24b0 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876621.948182:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xba:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876621.948211:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.948219:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.948225:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.948233:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.948240:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.948244:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.948247:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704838c 00000080:00010000:1.0:1788876621.948250:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704838c from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.948254:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.948261:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.948279:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.948285:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-144, flags=0 00000080:00200000:1.0:1788876621.948288:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.948294:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.948301:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.948309:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.948315:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.948319:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.948322:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704838c 00000080:00010000:1.0:1788876621.948325:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704838c from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.948328:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.948335:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.948343:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.948348:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-144, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.948370:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.948571:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a3400/0x7d14dcdc3704801a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xba:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab4e2 expref: -99 pid: 33293 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.948593:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xba:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.948625:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a3400/0x7d14dcdc3704801a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab4e2 expref: -99 pid: 33293 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.948633:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.948639:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.948642:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887a3400/0x7d14dcdc3704801a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xba:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab4e2 expref: -99 pid: 33293 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.948660:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887be00/0x7d14dcdc3704838c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac781 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.948676:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.948684:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898d880 x1875771511137664/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.948704:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898d880 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511137664:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.948943:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000080:00200000:1.0:1788876621.958057:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.958072:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.958083:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.958093:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876621.958115:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.958120:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.958123:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048393 00000080:00010000:1.0:1788876621.958129:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048393 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876621.958131:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876621.958138:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.958149:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876621.958158:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-145, flags=0 00000080:00200000:1.0:1788876621.958163:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876621.958169:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876621.958176:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.958184:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876621.958190:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876621.958193:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876621.958195:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048393 00000080:00010000:1.0:1788876621.958198:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048393 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876621.958200:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876621.958219:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.958227:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876621.958233:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-145, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876621.958258:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.958307:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188730000/0x7d14dcdc37048021 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xbb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab505 expref: -99 pid: 33294 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876621.958317:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xbb:0x0]: losing layout lock 00010000:00010000:1.0:1788876621.958354:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188730000/0x7d14dcdc37048021 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xbb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab505 expref: -99 pid: 33294 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.958362:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.958368:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876621.958371:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188730000/0x7d14dcdc37048021 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xbb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab505 expref: -99 pid: 33294 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876621.958571:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048393 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac7ab expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876621.958590:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876621.958602:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898e680 x1875771511138048/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876621.958634:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898e680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511138048:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876621.958829:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.076918:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511142400:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876622.076945:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.076958:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.076972:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.076983:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.076996:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.077128:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.077136:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876622.077149:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06198afce00 x1875771511142400/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876638 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876622.077182:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.077196:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876622.077202:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876622.077228:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876622.077234:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483e7 00000080:00010000:1.0:1788876622.077239:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.077242:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483e7 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876622.077245:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.077253:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.077280:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.077290:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-156, flags=8 00000080:00200008:1.0:1788876622.077300:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-156:[0x200001b71:0xc6:0x0](ffffa0619826e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876622.077305:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xc6:0x0](ffffa0619826e3c8),name=f5.replay-single-156 00000002:00010000:1.0:1788876622.077324:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xc6:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.077334:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887af800/0x7d14dcdc3704806e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab686 expref: -99 pid: 33305 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.077345:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887af800/0x7d14dcdc3704806e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab686 expref: -99 pid: 33305 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.077352:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.077362:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xc6:0x0](ffffa0619826e3c8) 00000080:00010000:1.0:1788876622.077364:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xc6:0x0](ffffa0619826e3c8) for lock 0x7d14dcdc3704806e 00000080:00010000:1.0:1788876622.077369:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704806e from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.077372:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887af800/0x7d14dcdc3704806e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xc6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab686 expref: -99 pid: 33305 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.077414:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887af800/0x7d14dcdc3704806e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab686 expref: -99 pid: 33305 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.077434:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc6:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876622.077443:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xc6:0x0] 00000008:00010000:1.0:1788876622.077462:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.077465:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb24b0, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.077481:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cbd50017/0x7d14dcdc3704768f lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x20f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a76de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.077493:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cbd50017/0x7d14dcdc3704768f lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a76de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.077504:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.077509:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cbd50017/0x7d14dcdc3704768f lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a76de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.077517:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cbd50017/0x7d14dcdc3704768f lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a76de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.077526:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.077536:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xc6:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876622.077544:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.077547:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.077550:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb24b0 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.077554:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.077557:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb24b0 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.077568:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc6:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.077596:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.077605:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.077611:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.077619:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.077625:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.077628:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.077631:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483e7 00000080:00010000:1.0:1788876622.077634:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483e7 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.077637:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.077643:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.077653:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.077659:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-156, flags=0 00000080:00200000:1.0:1788876622.077663:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.077669:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.077681:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.077689:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.077696:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.077700:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.077702:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483e7 00000080:00010000:1.0:1788876622.077706:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483e7 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.077709:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.077715:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.077801:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.077808:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-156, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.077831:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.078000:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887af800/0x7d14dcdc3704806e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc6:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab686 expref: -99 pid: 33305 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.078089:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xc6:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.078126:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887af800/0x7d14dcdc3704806e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xc6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab686 expref: -99 pid: 33305 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.078133:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.078144:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.078147:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887af800/0x7d14dcdc3704806e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xc6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab686 expref: -99 pid: 33305 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.078166:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370483e7 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac9a3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.078185:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.078194:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898ca80 x1875771511142656/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.078216:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898ca80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511142656:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.078300:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.081946:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898ca80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511142656:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876622.082088:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc6:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876622.082105:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876622.082134:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.082185:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xc6:0x0](ffffa0619826e3c8) 00000008:00010000:1.0:1788876622.082221:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876619 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cbd50017/0x7d14dcdc3704768f lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x20f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a76de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876622.082301:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.082319:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.082491:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.082516:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.082527:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876622.082546:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898ce00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511142784:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876622.082613:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.085127:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898ce00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511142784:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876622.085151:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085163:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085176:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085188:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085201:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085209:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085231:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876622.085243:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898ce00 x1875771511142784/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876638 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876622.085257:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085270:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876622.085276:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876622.085300:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876622.085306:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483ee 00000080:00010000:1.0:1788876622.085311:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.085314:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483ee from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876622.085318:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.085328:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.085353:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.085362:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-157, flags=8 00000080:00200008:1.0:1788876622.085373:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-157:[0x200001b71:0xc7:0x0](ffffa06198269148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876622.085552:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xc7:0x0](ffffa06198269148),name=f5.replay-single-157 00000002:00010000:1.0:1788876622.085571:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xc7:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.085582:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae600/0x7d14dcdc37048075 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6a9 expref: -99 pid: 33306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.085600:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae600/0x7d14dcdc37048075 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6a9 expref: -99 pid: 33306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.085607:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.085613:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xc7:0x0](ffffa06198269148) 00000080:00010000:1.0:1788876622.085616:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xc7:0x0](ffffa06198269148) for lock 0x7d14dcdc37048075 00000080:00010000:1.0:1788876622.085621:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048075 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.085623:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae600/0x7d14dcdc37048075 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xc7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab6a9 expref: -99 pid: 33306 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.085630:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae600/0x7d14dcdc37048075 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6a9 expref: -99 pid: 33306 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.085649:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc7:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876622.085658:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xc7:0x0] 00000008:00010000:1.0:1788876622.085677:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.085680:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.085695:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068a75003/0x7d14dcdc370476a4 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x210:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7716 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.085706:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068a75003/0x7d14dcdc370476a4 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7716 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.085717:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.085721:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068a75003/0x7d14dcdc370476a4 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7716 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.085805:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068a75003/0x7d14dcdc370476a4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7716 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.085817:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.085828:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xc7:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876622.085832:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.085835:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.085838:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.085841:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.085843:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.085916:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc7:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.085944:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.085953:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.085961:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.085969:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.085975:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.085979:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.085982:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483ee 00000080:00010000:1.0:1788876622.085985:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483ee from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.085989:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.086078:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.086088:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.086096:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-157, flags=0 00000080:00200000:1.0:1788876622.086100:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.086108:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.086115:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.086123:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.086129:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.086133:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.086135:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483ee 00000080:00010000:1.0:1788876622.086139:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483ee from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.086141:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.086148:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.086156:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.086162:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-157, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.086182:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.086233:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae600/0x7d14dcdc37048075 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc7:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab6a9 expref: -99 pid: 33306 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.086244:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xc7:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.086276:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae600/0x7d14dcdc37048075 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xc7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab6a9 expref: -99 pid: 33306 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.086283:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.086289:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.086292:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae600/0x7d14dcdc37048075 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xc7:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab6a9 expref: -99 pid: 33306 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.086309:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483ee lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac9cd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.086325:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.086333:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898ce00 x1875771511143040/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.086353:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898ce00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143040:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.086603:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.090232:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898ce00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143040:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876622.090291:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc7:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876622.090306:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876622.090335:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc7:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.090430:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xc7:0x0](ffffa06198269148) 00000008:00010000:1.0:1788876622.090464:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876619 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068a75003/0x7d14dcdc370476a4 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x210:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7716 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876622.090538:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.090558:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.090614:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.090623:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.090634:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876622.090653:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898ea00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143168:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876622.090736:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.093921:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898ea00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143168:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876622.093948:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.093960:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.093973:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.093984:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.094023:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.094032:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.094040:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876622.094053:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898ea00 x1875771511143168/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876638 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876622.094068:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.094082:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876622.094089:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876622.094121:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876622.094128:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483f5 00000080:00010000:1.0:1788876622.094133:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.094136:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483f5 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876622.094139:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.094148:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.094185:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.094197:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-158, flags=8 00000080:00200008:1.0:1788876622.094208:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-158:[0x200001b71:0xc8:0x0](ffffa0619826aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876622.094214:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xc8:0x0](ffffa0619826aa08),name=f5.replay-single-158 00000002:00010000:1.0:1788876622.094235:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xc8:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.094246:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887afc00/0x7d14dcdc3704807c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6cc expref: -99 pid: 33307 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.094258:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887afc00/0x7d14dcdc3704807c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6cc expref: -99 pid: 33307 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.094264:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.094268:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xc8:0x0](ffffa0619826aa08) 00000080:00010000:1.0:1788876622.094271:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xc8:0x0](ffffa0619826aa08) for lock 0x7d14dcdc3704807c 00000080:00010000:1.0:1788876622.094274:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704807c from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.094278:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887afc00/0x7d14dcdc3704807c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xc8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab6cc expref: -99 pid: 33307 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.094285:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887afc00/0x7d14dcdc3704807c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6cc expref: -99 pid: 33307 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.094305:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc8:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876622.094316:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xc8:0x0] 00000008:00010000:1.0:1788876622.094337:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.094340:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.094358:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.094417:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.094431:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.094435:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.094443:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.094454:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.094466:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xc8:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876622.094470:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.094473:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.094476:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.094481:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.094483:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.094494:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc8:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.094523:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.094532:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.094540:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.094547:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.094564:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.094568:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.094571:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483f5 00000080:00010000:1.0:1788876622.094575:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483f5 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.094578:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.094585:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.094593:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.094600:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-158, flags=0 00000080:00200000:1.0:1788876622.094603:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.094609:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.094615:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.094622:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.094627:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.094631:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.094633:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483f5 00000080:00010000:1.0:1788876622.094635:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483f5 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.094638:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.094654:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.094661:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.094666:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-158, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.094686:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.094739:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887afc00/0x7d14dcdc3704807c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc8:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab6cc expref: -99 pid: 33307 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.094749:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xc8:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.094782:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887afc00/0x7d14dcdc3704807c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xc8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab6cc expref: -99 pid: 33307 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.094789:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.094794:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.094797:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887afc00/0x7d14dcdc3704807c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xc8:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab6cc expref: -99 pid: 33307 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.094814:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370483f5 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ac9f7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.094832:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.094839:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898ea00 x1875771511143424/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.094867:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898ea00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143424:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.094927:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.100703:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898ea00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143424:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876622.100845:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc8:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876622.100928:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876622.100958:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc8:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.101020:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xc8:0x0](ffffa0619826aa08) 00000008:00010000:1.0:1788876622.101060:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876619 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876622.101151:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.101174:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.101237:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.101246:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.101254:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876622.101276:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143552:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876622.101644:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.104340:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143552:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876622.104372:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.104654:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.104669:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.104682:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.104696:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.104706:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.104714:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876622.104797:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898dc00 x1875771511143552/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876638 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876622.104812:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.104827:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876622.107845:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876622.108028:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876622.108055:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483fc 00000080:00010000:1.0:1788876622.108063:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.108066:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483fc from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876622.108073:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.108087:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.108119:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.108132:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-159, flags=8 00000080:00200008:1.0:1788876622.108143:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-159:[0x200001b71:0xc9:0x0](ffffa0619826db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876622.108149:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xc9:0x0](ffffa0619826db88),name=f5.replay-single-159 00000002:00010000:1.0:1788876622.108171:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xc9:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.108188:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae200/0x7d14dcdc37048083 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6ef expref: -99 pid: 33308 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.108200:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae200/0x7d14dcdc37048083 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xc9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6ef expref: -99 pid: 33308 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.108207:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.108212:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xc9:0x0](ffffa0619826db88) 00000080:00010000:1.0:1788876622.108215:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xc9:0x0](ffffa0619826db88) for lock 0x7d14dcdc37048083 00000080:00010000:1.0:1788876622.108218:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048083 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.108222:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae200/0x7d14dcdc37048083 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xc9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab6ef expref: -99 pid: 33308 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.108229:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae200/0x7d14dcdc37048083 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab6ef expref: -99 pid: 33308 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.108258:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc9:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876622.108269:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xc9:0x0] 00000008:00010000:1.0:1788876622.108291:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.108295:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.108313:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001dc9cc83/0x7d14dcdc370476ce lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x211:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7786 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.108326:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001dc9cc83/0x7d14dcdc370476ce lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7786 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.108339:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.108343:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001dc9cc83/0x7d14dcdc370476ce lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7786 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.108350:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001dc9cc83/0x7d14dcdc370476ce lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7786 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.108361:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.108373:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xc9:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876622.108377:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.112254:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.112263:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.112269:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.112272:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.112292:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc9:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.112341:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.112402:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.112425:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.112441:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.112449:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.112456:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.112460:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483fc 00000080:00010000:1.0:1788876622.112466:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483fc from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.112470:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.112478:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.112493:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.112503:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-159, flags=0 00000080:00200000:1.0:1788876622.112507:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.112513:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.112519:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.112527:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.112532:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.112536:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.112544:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370483fc 00000080:00010000:1.0:1788876622.112548:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370483fc from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.112551:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.112558:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.112568:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.112573:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-159, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.112603:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.112651:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae200/0x7d14dcdc37048083 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xc9:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab6ef expref: -99 pid: 33308 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.112661:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xc9:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.112701:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae200/0x7d14dcdc37048083 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xc9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab6ef expref: -99 pid: 33308 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.112709:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.112715:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.112718:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae200/0x7d14dcdc37048083 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xc9:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab6ef expref: -99 pid: 33308 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.112736:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370483fc lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aca21 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.112770:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.112783:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898dc00 x1875771511143808/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.112808:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143808:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.112891:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.116326:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898dc00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143808:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876622.116416:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xc9:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876622.116430:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876622.116457:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xc9:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.116506:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xc9:0x0](ffffa0619826db88) 00000008:00010000:1.0:1788876622.116540:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876619 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001dc9cc83/0x7d14dcdc370476ce lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x211:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7786 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876622.116618:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.116637:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.116698:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.116707:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.116716:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876622.116747:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143936:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876622.116822:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.119361:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511143936:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876622.119451:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119463:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119475:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119486:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119499:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119508:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119515:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876622.119526:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618898c380 x1875771511143936/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876638 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876622.119539:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119571:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876622.119577:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876622.119598:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876622.119603:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048403 00000080:00010000:1.0:1788876622.119609:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.119612:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048403 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876622.119615:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.119625:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.119647:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.119657:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-16, flags=8 00000080:00200008:1.0:1788876622.119668:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-16:[0x200001b71:0x3a:0x0](ffffa061982263c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876622.119672:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x3a:0x0](ffffa061982263c8),name=f5.replay-single-16 00000002:00010000:1.0:1788876622.119688:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x3a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.119697:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825c00/0x7d14dcdc37047c9a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa362 expref: -99 pid: 33164 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.119707:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825c00/0x7d14dcdc37047c9a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa362 expref: -99 pid: 33164 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.119713:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.119717:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x3a:0x0](ffffa061982263c8) 00000080:00010000:1.0:1788876622.119720:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x3a:0x0](ffffa061982263c8) for lock 0x7d14dcdc37047c9a 00000080:00010000:1.0:1788876622.119742:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047c9a from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.119746:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825c00/0x7d14dcdc37047c9a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa362 expref: -99 pid: 33164 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.119754:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825c00/0x7d14dcdc37047c9a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa362 expref: -99 pid: 33164 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.119772:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x3a:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876622.119782:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x3a:0x0] 00000008:00010000:1.0:1788876622.119802:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.119805:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.119822:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007f4cb5f6/0x7d14dcdc37046b13 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a583e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.119834:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007f4cb5f6/0x7d14dcdc37046b13 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a583e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.119845:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.119849:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007f4cb5f6/0x7d14dcdc37046b13 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a583e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.119946:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007f4cb5f6/0x7d14dcdc37046b13 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1c9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a583e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.119957:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.119967:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x3a:0x0] updating i_size 7 i_blocks 0 00000020:00010000:1.0:1788876622.119973:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.119975:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.119984:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.119988:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.119991:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.120003:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x3a:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.120045:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.120054:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.120061:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.120070:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.120076:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.120080:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.120083:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048403 00000080:00010000:1.0:1788876622.120087:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048403 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.120091:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.120098:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.120106:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.120114:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-16, flags=0 00000080:00200000:1.0:1788876622.120119:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.120126:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.120132:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.120151:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.120157:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.120161:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.120163:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048403 00000080:00010000:1.0:1788876622.120167:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048403 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.120169:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.120177:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.120184:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.120189:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-16, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.120210:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.120252:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825c00/0x7d14dcdc37047c9a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa362 expref: -99 pid: 33164 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.120263:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x3a:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.120299:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825c00/0x7d14dcdc37047c9a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa362 expref: -99 pid: 33164 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.120307:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.120323:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.120326:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825c00/0x7d14dcdc37047c9a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x3a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa362 expref: -99 pid: 33164 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.120344:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048403 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aca4b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.120362:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.120371:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618898c380 x1875771511144192/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.120527:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144192:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.120605:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.126782:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618898c380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144192:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876622.126839:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x3a:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876622.126850:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876622.126884:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x3a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.126925:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x3a:0x0](ffffa061982263c8) 00000008:00010000:1.0:1788876622.126956:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 0 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 0 [cma]time 1788876559 1788876559 1788876613 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007f4cb5f6/0x7d14dcdc37046b13 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1c9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a583e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876622.127036:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.127055:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.127109:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.127131:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.127142:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876622.127160:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144320:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876622.127228:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.129819:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144320:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876622.129842:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.129864:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.129877:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.129888:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.129901:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.129909:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.129933:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876622.129944:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188845c00 x1875771511144320/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876638 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876622.129958:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.129970:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876622.129976:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876622.129998:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876622.130004:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704840a 00000080:00010000:1.0:1788876622.130008:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.130021:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704840a from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876622.130025:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.130035:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.130058:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.130068:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-160, flags=8 00000080:00200008:1.0:1788876622.130077:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-160:[0x200001b71:0xca:0x0](ffffa0619826a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876622.130082:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xca:0x0](ffffa0619826a1c8),name=f5.replay-single-160 00000002:00010000:1.0:1788876622.130100:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xca:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.130110:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae400/0x7d14dcdc3704808a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab712 expref: -99 pid: 33309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.130127:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae400/0x7d14dcdc3704808a lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xca:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab712 expref: -99 pid: 33309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.130134:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.130139:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xca:0x0](ffffa0619826a1c8) 00000080:00010000:1.0:1788876622.130142:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xca:0x0](ffffa0619826a1c8) for lock 0x7d14dcdc3704808a 00000080:00010000:1.0:1788876622.130146:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704808a from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.130149:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae400/0x7d14dcdc3704808a lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xca:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab712 expref: -99 pid: 33309 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.130156:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae400/0x7d14dcdc3704808a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xca:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab712 expref: -99 pid: 33309 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.130175:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xca:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876622.130184:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xca:0x0] 00000008:00010000:1.0:1788876622.130201:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.130205:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.130220:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000dea3658c/0x7d14dcdc370476e3 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x211:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a77be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.130231:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000dea3658c/0x7d14dcdc370476e3 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a77be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.130241:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.130245:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000dea3658c/0x7d14dcdc370476e3 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a77be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.130263:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000dea3658c/0x7d14dcdc370476e3 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a77be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.130273:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.130284:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xca:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876622.130287:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.130290:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.130292:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.130296:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.130299:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.130308:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xca:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.130338:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.130346:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.130353:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.130362:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.130368:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.130372:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.130375:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704840a 00000080:00010000:1.0:1788876622.130379:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704840a from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.130414:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.130431:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.130440:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.130448:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-160, flags=0 00000080:00200000:1.0:1788876622.130452:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.130460:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.130466:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.130475:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.130481:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.130485:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.130487:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704840a 00000080:00010000:1.0:1788876622.130491:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704840a from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.130494:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.130501:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.130509:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.130514:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-160, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.130536:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.130585:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae400/0x7d14dcdc3704808a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xca:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab712 expref: -99 pid: 33309 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.130596:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xca:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.130628:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae400/0x7d14dcdc3704808a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xca:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab712 expref: -99 pid: 33309 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.130637:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.130643:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.130645:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ae400/0x7d14dcdc3704808a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xca:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab712 expref: -99 pid: 33309 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.130661:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc3704840a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aca75 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.130676:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.130683:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188845c00 x1875771511144576/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.130702:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144576:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.130752:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.135092:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144576:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876622.135151:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xca:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876622.135164:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876622.135190:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xca:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.135248:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0xca:0x0](ffffa0619826a1c8) 00000008:00010000:1.0:1788876622.135282:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 8 blocks 8 [cma]time: 1788876562 1788876562 0 by oinfo size 8 blocks 8 [cma]time 1788876562 1788876562 1788876619 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000dea3658c/0x7d14dcdc370476e3 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x211:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a77be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876622.135353:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.135369:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.135451:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.135460:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.135469:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876622.135487:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144704:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876622.135539:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000100:00100000:1.0:1788876622.138081:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511144704:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876622.138104:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138116:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138129:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138140:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138166:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138175:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138182:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876622.138193:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188846d80 x1875771511144704/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876638 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876622.138208:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138220:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876622.138226:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876622.138251:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876622.138257:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048411 00000080:00010000:1.0:1788876622.138263:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.138266:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048411 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876622.138270:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876622.138279:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048411 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aca9f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.138313:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.138323:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-161, flags=8 00000080:00200008:1.0:1788876622.138332:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-161:[0x200001b71:0xcb:0x0](ffffa0619826d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876622.138336:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xcb:0x0](ffffa0619826d348),name=f5.replay-single-161 00000002:00010000:1.0:1788876622.138353:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xcb:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.138363:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baae00/0x7d14dcdc37048091 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xcb:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab735 expref: -99 pid: 33310 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.138373:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baae00/0x7d14dcdc37048091 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xcb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab735 expref: -99 pid: 33310 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.138603:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.138609:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xcb:0x0](ffffa0619826d348) 00000080:00010000:1.0:1788876622.138611:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xcb:0x0](ffffa0619826d348) for lock 0x7d14dcdc37048091 00000080:00010000:1.0:1788876622.138618:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048091 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.138621:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baae00/0x7d14dcdc37048091 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xcb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ab735 expref: -99 pid: 33310 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.138628:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187baae00/0x7d14dcdc37048091 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xcb:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ab735 expref: -99 pid: 33310 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.242408:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x3b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876622.242422:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x3b:0x0] 00000008:00010000:1.0:1788876622.242451:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.242461:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.242483:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009bf36839/0x7d14dcdc37046b28 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1ca:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5876 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.242505:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009bf36839/0x7d14dcdc37046b28 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5876 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.242517:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.242521:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009bf36839/0x7d14dcdc37046b28 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5876 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.242529:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009bf36839/0x7d14dcdc37046b28 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5876 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.242542:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.242553:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x3b:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876622.242557:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.242560:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.242563:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.242567:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.242569:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.242580:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x3b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.242614:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.242626:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.242635:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.242644:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.242664:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.242670:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.242673:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048450 00000080:00010000:1.0:1788876622.242678:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048450 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.242681:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.242688:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.242698:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.242717:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-17, flags=0 00000080:00200000:1.0:1788876622.242723:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.242730:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.242737:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.242744:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.242751:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.242754:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.242757:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048450 00000080:00010000:1.0:1788876622.242760:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048450 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.242763:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.242779:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.242788:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.242793:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-17, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.242817:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.242857:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188824200/0x7d14dcdc37047ca1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa385 expref: -99 pid: 33165 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.242868:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x3b:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.242902:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188824200/0x7d14dcdc37047ca1 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa385 expref: -99 pid: 33165 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.242910:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.242916:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.242919:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188824200/0x7d14dcdc37047ca1 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x3b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa385 expref: -99 pid: 33165 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.242942:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cf000/0x7d14dcdc37048450 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313acc19 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.242959:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.242968:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188846d80 x1875771511148416/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.242998:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511148416:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.243084:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64f00 going to sleep for 11 seconds 00000080:00200000:1.0:1788876622.258438:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.258456:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.258471:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.258483:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.258491:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.258497:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.258499:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048457 00000080:00010000:1.0:1788876622.258504:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048457 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.258508:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.258515:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.258527:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.258535:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-170, flags=0 00000080:00200000:1.0:1788876622.258540:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.258545:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.258550:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.258572:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.258578:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.258582:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.258584:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048457 00000080:00010000:1.0:1788876622.258588:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048457 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.258590:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.258598:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.258606:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.258610:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-170, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.258636:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.258683:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.258693:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xd4:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.263228:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.263250:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.263278:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.263282:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061813a1c00/0x7d14dcdc370480d0 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xd4:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ab870 expref: -99 pid: 33319 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.263307:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc37048457 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313acc43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.263327:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.263339:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188847480 x1875771511148800/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.263364:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511148800:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.263477:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000080:00200000:1.0:1788876622.573461:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0xe5:0x0](ffffa06198263248),name=f5.replay-single-187 00000002:00010000:1.0:1788876622.573487:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0xe5:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.573511:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840e00/0x7d14dcdc37048147 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abac3 expref: -99 pid: 33336 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.573524:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840e00/0x7d14dcdc37048147 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abac3 expref: -99 pid: 33336 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.573531:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.573536:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0xe5:0x0](ffffa06198263248) 00000080:00010000:1.0:1788876622.573539:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0xe5:0x0](ffffa06198263248) for lock 0x7d14dcdc37048147 00000080:00010000:1.0:1788876622.573543:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048147 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876622.573546:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840e00/0x7d14dcdc37048147 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313abac3 expref: -99 pid: 33336 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.573561:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840e00/0x7d14dcdc37048147 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313abac3 expref: -99 pid: 33336 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.573582:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876622.573592:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0xe5:0x0] 00000008:00010000:1.0:1788876622.573613:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876622.573616:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876622.573635:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000061125bd1/0x7d14dcdc3704791a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x21f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7da6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.573647:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000061125bd1/0x7d14dcdc3704791a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7da6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876622.573659:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876622.573663:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000061125bd1/0x7d14dcdc3704791a lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a7da6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876622.573671:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000061125bd1/0x7d14dcdc3704791a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x21f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a7da6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876622.573681:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876622.573691:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0xe5:0x0] updating i_size 8 i_blocks 8 00000020:00010000:1.0:1788876622.573695:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876622.573698:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.573701:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876622.573704:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876622.573713:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876622.573736:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0xe5:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876622.573768:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.573777:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.573785:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.573793:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876622.573799:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.573803:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.573806:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370484d5 00000080:00010000:1.0:1788876622.573811:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370484d5 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876622.573814:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876622.573821:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.573831:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876622.573838:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-187, flags=0 00000080:00200000:1.0:1788876622.573843:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876622.573862:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876622.573869:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.573878:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876622.573893:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876622.573897:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876622.573900:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370484d5 00000080:00010000:1.0:1788876622.573904:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370484d5 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876622.573907:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876622.573915:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.573924:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876622.573929:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-187, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876622.573953:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.573992:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840e00/0x7d14dcdc37048147 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313abac3 expref: -99 pid: 33336 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876622.574004:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0xe5:0x0]: losing layout lock 00010000:00010000:1.0:1788876622.574049:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840e00/0x7d14dcdc37048147 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313abac3 expref: -99 pid: 33336 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.574057:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.574063:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876622.574066:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188840e00/0x7d14dcdc37048147 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0xe5:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313abac3 expref: -99 pid: 33336 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876622.574089:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce000/0x7d14dcdc370484d5 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313acf37 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876622.574106:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876622.574113:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188847b80 x1875771511155712/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876622.574134:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847b80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511155712:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876622.574219:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00010000:00010000:1.0:1788876622.590373:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.104197:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511171200:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.104225:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104238:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104253:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104266:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104279:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104308:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104317:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.104329:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188961f80 x1875771511171200/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.104345:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104358:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.104364:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.104421:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.104428:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370485f4 00000080:00010000:1.0:1788876623.104433:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.104437:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370485f4 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.104440:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.104451:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.104477:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.104488:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-25, flags=8 00000080:00200008:1.0:1788876623.104500:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-25:[0x200001b71:0x43:0x0](ffffa0619822e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.104511:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x43:0x0](ffffa0619822e3c8),name=f5.replay-single-25 00000002:00010000:1.0:1788876623.104533:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x43:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.104544:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889adc00/0x7d14dcdc37047cd9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa49d expref: -99 pid: 33173 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.104556:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889adc00/0x7d14dcdc37047cd9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa49d expref: -99 pid: 33173 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.104564:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.104569:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x43:0x0](ffffa0619822e3c8) 00000080:00010000:1.0:1788876623.104572:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x43:0x0](ffffa0619822e3c8) for lock 0x7d14dcdc37047cd9 00000080:00010000:1.0:1788876623.104577:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047cd9 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.104580:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889adc00/0x7d14dcdc37047cd9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa49d expref: -99 pid: 33173 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.104589:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889adc00/0x7d14dcdc37047cd9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa49d expref: -99 pid: 33173 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.104609:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x43:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:1.0:1788876623.104619:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x43:0x0] 00000008:00010000:1.0:1788876623.104639:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.104643:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2a30, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.104661:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.104673:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.104698:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.104702:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.104711:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.104722:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.104733:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x43:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.104737:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.104740:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.104744:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2a30 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.104748:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.104751:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2a30 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.104762:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x43:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.104791:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.104799:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.104807:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.104816:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.104823:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.104827:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.104830:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370485f4 00000080:00010000:1.0:1788876623.104838:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370485f4 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.105193:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.105206:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.105216:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.105224:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-25, flags=0 00000080:00200000:1.0:1788876623.105228:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.105237:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.105244:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.105252:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.105258:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.105262:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.105264:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370485f4 00000080:00010000:1.0:1788876623.105268:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370485f4 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.105271:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.105278:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.105286:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.105296:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-25, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.105321:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.105355:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889adc00/0x7d14dcdc37047cd9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa49d expref: -99 pid: 33173 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.105365:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x43:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.105420:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889adc00/0x7d14dcdc37047cd9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa49d expref: -99 pid: 33173 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.105428:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.105434:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.105437:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889adc00/0x7d14dcdc37047cd9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x43:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa49d expref: -99 pid: 33173 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.105454:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370485f4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ad5f1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.105470:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.105479:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188844000 x1875771511171456/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.105500:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511171456:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.105610:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.130217:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.130290:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.135440:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.135462:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.135487:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532228736 00000100:00100000:1.0:1788876623.135494:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532228736 00000100:00100000:1.0:1788876623.135530:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532228864 00000100:00100000:1.0:1788876623.135533:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532228864 00000100:00100000:1.0:1788876623.135546:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 21 00000100:00100000:1.0:1788876623.135556:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188847100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8456:x1875771532228736:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.135576:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000058805c0c/0x7d14dcdc3704720c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6ad6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876623.135662:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.135669:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188847100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8456:x1875771532228736:12345-192.168.203.111@tcp:104: Request processed in 114us (238us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.135676:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 21 00000100:00100000:1.0:1788876623.135693:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 22 00000100:00100000:1.0:1788876623.135700:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188847800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43952:x1875771532228864:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.135709:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000018c2862c/0x7d14dcdc370472b4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6c96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.135740:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.135746:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188847800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43952:x1875771532228864:12345-192.168.203.111@tcp:104: Request processed in 46us (284us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.135751:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 22 00010000:00010000:1.0:1788876623.135769:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876623.135777:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000058805c0c/0x7d14dcdc3704720c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6ad6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876623.135804:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061887a2600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876623.135811:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000058805c0c/0x7d14dcdc3704720c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6ad6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876623.135869:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000058805c0c/0x7d14dcdc3704720c lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6ad6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876623.135877:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876623.135898:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000058805c0c/0x7d14dcdc3704720c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6ad6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876623.135908:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000058805c0c/0x7d14dcdc3704720c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1f4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6ad6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876623.135930:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876623.135934:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000018c2862c/0x7d14dcdc370472b4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6c96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.135942:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061887a3200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876623.135947:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000018c2862c/0x7d14dcdc370472b4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6c96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.135967:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000018c2862c/0x7d14dcdc370472b4 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6c96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.135990:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876623.135998:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000018c2862c/0x7d14dcdc370472b4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6c96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.136017:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000018c2862c/0x7d14dcdc370472b4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1f8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6c96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.136033:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876623.136035:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876623.136037:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.136049:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511172608:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.152983:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511172608:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.156016:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.156069:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.156094:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532231168 00000100:00100000:1.0:1788876623.156103:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532231168 00000100:00100000:1.0:1788876623.156139:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532231296 00000100:00100000:1.0:1788876623.156143:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532231296 00000100:00100000:1.0:1788876623.156158:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 31 00000100:00100000:1.0:1788876623.156168:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:43954:x1875771532231168:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.156186:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000837e8e6e/0x7d14dcdc37047308 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6d76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.156255:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.156261:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:43954:x1875771532231168:12345-192.168.203.111@tcp:104: Request processed in 95us (269us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.156279:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 31 00000100:00100000:1.0:1788876623.156295:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 32 00000100:00100000:1.0:1788876623.156302:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43956:x1875771532231296:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.156312:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000097072fa7/0x7d14dcdc37047386 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1fd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6ec6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.156360:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.156366:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43956:x1875771532231296:12345-192.168.203.111@tcp:104: Request processed in 65us (298us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.156372:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 32 00000100:00100000:1.0:1788876623.168427:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511173632:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.168446:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511173504:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.168476:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 34 00000100:00100000:1.0:1788876623.168485:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190ea9050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:10319:x1875771532231808:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.168495:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008d258bb6/0x7d14dcdc3704735c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6e56 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.168533:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.168543:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190ea9050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:10319:x1875771532231808:12345-192.168.203.111@tcp:104: Request processed in 57us (446us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.168550:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 34 00000100:00100000:1.0:1788876623.170188:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.170215:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532232320 00000100:00100000:1.0:1788876623.170220:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532232320 00000100:00100000:1.0:1788876623.170264:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 35 00000100:00100000:1.0:1788876623.170278:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:43958:x1875771532232320:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.170294:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003f900b45/0x7d14dcdc370473da lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6fa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.171809:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.171826:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:43958:x1875771532232320:12345-192.168.203.111@tcp:104: Request processed in 1544us (1640us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.171837:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 35 00000100:00100000:1.0:1788876623.171872:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 36 00000100:00100000:1.0:1788876623.171887:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190ecea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8455:x1875771532232192:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.171904:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ad91865b/0x7d14dcdc370473b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6f36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173123:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.173131:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190ecea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8455:x1875771532232192:12345-192.168.203.111@tcp:104: Request processed in 1248us (4273us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.173138:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 36 00000100:00100000:1.0:1788876623.173155:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532232576 00000100:00100000:1.0:1788876623.173162:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532232576 00000100:00100000:1.0:1788876623.173184:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532232704 00000100:00100000:1.0:1788876623.173186:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532232704 00000100:00100000:1.0:1788876623.173197:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 37 00000100:00100000:1.0:1788876623.173204:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190ecfb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8456:x1875771532232576:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.173217:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005603e218/0x7d14dcdc3704731d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6dae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173273:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.173279:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190ecfb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:8456:x1875771532232576:12345-192.168.203.111@tcp:104: Request processed in 75us (639us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.173285:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 37 00000100:00100000:1.0:1788876623.173297:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 38 00000100:00100000:1.0:1788876623.173303:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190ecf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:17045:x1875771532232704:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.173312:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173343:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876623.173348:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190ecf100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:17045:x1875771532232704:12345-192.168.203.111@tcp:104: Request processed in 45us (401us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.173354:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 38 00010000:00010000:1.0:1788876623.173367:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876623.173373:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ad91865b/0x7d14dcdc370473b0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6f36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173407:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886ed200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876623.173413:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ad91865b/0x7d14dcdc370473b0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6f36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173446:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ad91865b/0x7d14dcdc370473b0 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6f36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173454:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876623.173477:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ad91865b/0x7d14dcdc370473b0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6f36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173486:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ad91865b/0x7d14dcdc370473b0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6f36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173502:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876623.173506:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005603e218/0x7d14dcdc3704731d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6dae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173513:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061887afa00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876623.173517:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005603e218/0x7d14dcdc3704731d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6dae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173538:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005603e218/0x7d14dcdc3704731d lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6dae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173545:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876623.173551:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005603e218/0x7d14dcdc3704731d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6dae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173558:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005603e218/0x7d14dcdc3704731d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1fa:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6dae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173571:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876623.173574:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173594:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886ede00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876623.173598:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173617:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173623:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876623.173628:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173636:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000093ec924b/0x7d14dcdc37047404 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7016 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.173648:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876623.173650:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876623.173652:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.173666:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511174656:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.173706:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511174912:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.173863:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788876623.173871:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511174784:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876623.173918:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.180840:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.180895:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.180941:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532233088 00000100:00100000:1.0:1788876623.180951:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532233088 00000100:00100000:1.0:1788876623.180996:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532233344 00000100:00100000:1.0:1788876623.180999:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532233344 00000100:00100000:1.0:1788876623.182721:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 41 00000100:00100000:1.0:1788876623.182732:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43954:x1875771532233344:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.182897:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000097b4990b/0x7d14dcdc37047371 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1fc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6e8e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.183224:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.183235:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43954:x1875771532233344:12345-192.168.203.111@tcp:104: Request processed in 502us (2337us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.183244:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 41 00010000:00010000:1.0:1788876623.188322:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.188605:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511174784:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.188620:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511174656:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.188629:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511174912:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.193132:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 42 00000100:00100000:1.0:1788876623.193146:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190ecea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8455:x1875771532233728:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.193169:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000075e4e898/0x7d14dcdc37046abf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1c7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a575e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1788876623.193285:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.193292:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190ecea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8455:x1875771532233728:12345-192.168.203.111@tcp:104: Request processed in 151us (1052us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.193317:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 42 00000100:00100000:1.0:1788876623.193379:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.193427:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532233984 00000100:00100000:1.0:1788876623.193431:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532233984 00000100:00100000:1.0:1788876623.193453:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 43 00000100:00100000:1.0:1788876623.193459:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:10319:x1875771532233984:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.193469:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b3d2caaf/0x7d14dcdc370473c5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1fe:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6f6e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.198665:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.198680:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:10319:x1875771532233984:12345-192.168.203.111@tcp:104: Request processed in 5214us (5294us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.198692:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 43 00000100:00100000:1.0:1788876623.198829:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.198866:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532234112 00000100:00100000:1.0:1788876623.198874:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532234112 00000100:00100000:1.0:1788876623.198912:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 45 00000100:00100000:1.0:1788876623.198920:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8456:x1875771532234112:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.198937:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d83218ed/0x7d14dcdc370473ef lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ff:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6fde expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.202252:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.202262:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8456:x1875771532234112:12345-192.168.203.111@tcp:104: Request processed in 3342us (3432us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.202273:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 45 00000100:00100000:1.0:1788876623.202418:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.202451:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532234368 00000100:00100000:1.0:1788876623.202459:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532234368 00000100:00100000:1.0:1788876623.202493:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 46 00000100:00100000:1.0:1788876623.202501:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:17045:x1875771532234368:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.202514:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004b69d3e7/0x7d14dcdc37047458 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a70f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.202567:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.202572:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:17045:x1875771532234368:12345-192.168.203.111@tcp:104: Request processed in 72us (155us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.202578:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 46 00000100:00100000:1.0:1788876623.202981:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.202996:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532234752 00000100:00100000:1.0:1788876623.202999:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532234752 00000100:00100000:1.0:1788876623.203703:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 47 00000100:00100000:1.0:1788876623.203711:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8453:x1875771532234752:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.204508:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000028d93223/0x7d14dcdc37046ad4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1c8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5796 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.204551:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.204558:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8453:x1875771532234752:12345-192.168.203.111@tcp:104: Request processed in 847us (1578us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.204566:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 47 00000100:00100000:1.0:1788876623.204621:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.204647:0:15182:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876623.204656:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532235008 00000100:00100000:1.0:1788876623.204661:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532235008 00000100:00100000:1.0:1788876623.204683:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 48 00000100:00100000:1.0:1788876623.204702:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190eaac50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43952:x1875771532235008:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.204710:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000cff0eb23/0x7d14dcdc37047419 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x200:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a704e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.204941:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.204947:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190eaac50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43952:x1875771532235008:12345-192.168.203.111@tcp:104: Request processed in 246us (326us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.204954:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 48 00000100:00100000:1.0:1788876623.205080:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.205097:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532235264 00000100:00100000:1.0:1788876623.205100:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532235264 00000100:00100000:1.0:1788876623.205120:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 49 00000100:00100000:1.0:1788876623.205126:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532235264:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.205135:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f2d8e4dc/0x7d14dcdc37047482 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7166 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.205161:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876623.205166:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532235264:12345-192.168.203.111@tcp:104: Request processed in 40us (87us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.205172:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 49 00000100:00100000:1.0:1788876623.205219:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.205235:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532235392 00000100:00100000:1.0:1788876623.205239:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532235392 00000100:00100000:1.0:1788876623.205257:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 50 00000100:00100000:1.0:1788876623.205263:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43960:x1875771532235392:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.205272:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000083e8027b/0x7d14dcdc37047443 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x201:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a70be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.205310:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1788876623.205315:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43960:x1875771532235392:12345-192.168.203.111@tcp:104: Request processed in 53us (97us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.205321:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 50 00000100:00100000:1.0:1788876623.209922:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.209953:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532235520 00000100:00100000:1.0:1788876623.209960:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532235520 00000100:00100000:1.0:1788876623.209998:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 51 00000100:00100000:1.0:1788876623.210021:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43954:x1875771532235520:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.210038:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000068119bad/0x7d14dcdc370474ac lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a71d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.210095:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.210101:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43954:x1875771532235520:12345-192.168.203.111@tcp:104: Request processed in 82us (185us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.210107:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 51 00000100:00100000:1.0:1788876623.210170:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.210182:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532235776 00000100:00100000:1.0:1788876623.210185:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532235776 00000100:00100000:1.0:1788876623.210197:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 52 00000100:00100000:1.0:1788876623.210202:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43953:x1875771532235776:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.210211:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d7a262a9/0x7d14dcdc370474d6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7246 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.210248:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.210266:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43953:x1875771532235776:12345-192.168.203.111@tcp:104: Request processed in 63us (95us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.210272:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 52 00000100:00100000:1.0:1788876623.210358:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.210372:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532236544 00000100:00100000:1.0:1788876623.210375:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532236544 00000100:00100000:1.0:1788876623.210811:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 53 00000100:00100000:1.0:1788876623.210818:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43961:x1875771532236544:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.210827:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000006c61f8c8/0x7d14dcdc37047497 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x203:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a719e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.210933:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.210939:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43961:x1875771532236544:12345-192.168.203.111@tcp:104: Request processed in 122us (581us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.210946:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 53 00000100:00100000:1.0:1788876623.210987:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.211358:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532236672 00000100:00100000:1.0:1788876623.211363:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532236672 00000100:00100000:1.0:1788876623.211408:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 54 00000100:00100000:1.0:1788876623.211415:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188847100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:10319:x1875771532236672:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.211424:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000023784b25/0x7d14dcdc37047500 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a72b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.211461:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.211467:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188847100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:10319:x1875771532236672:12345-192.168.203.111@tcp:104: Request processed in 53us (480us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.211473:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 54 00000100:00100000:1.0:1788876623.211525:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.211544:0:15182:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/3 left), rc = 0 00000100:00100000:1.0:1788876623.211554:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532236416 00000100:00100000:1.0:1788876623.211556:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532236416 00000100:00100000:1.0:1788876623.211569:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 55 00000100:00100000:1.0:1788876623.211574:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061889c5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43959:x1875771532236416:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.211582:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000d6fb260/0x7d14dcdc3704746d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x202:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a712e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.211604:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876623.211609:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061889c5850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43959:x1875771532236416:12345-192.168.203.111@tcp:104: Request processed in 35us (84us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.211614:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 55 00000100:00100000:1.0:1788876623.212495:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.212538:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532236928 00000100:00100000:1.0:1788876623.212543:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532236928 00000100:00100000:1.0:1788876623.212569:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 56 00000100:00100000:1.0:1788876623.212576:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8455:x1875771532236928:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.212588:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000abd09a5b/0x7d14dcdc3704752a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7326 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.212623:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1788876623.212629:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8455:x1875771532236928:12345-192.168.203.111@tcp:104: Request processed in 53us (140us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.212636:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 56 00000100:00100000:1.0:1788876623.219444:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.219475:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532237184 00000100:00100000:1.0:1788876623.219498:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532237184 00000100:00100000:1.0:1788876623.219549:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 57 00000100:00100000:1.0:1788876623.219561:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8456:x1875771532237184:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.219581:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ff819d25/0x7d14dcdc37047554 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7396 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.219653:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.219661:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8456:x1875771532237184:12345-192.168.203.111@tcp:104: Request processed in 102us (227us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.219668:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 57 00000100:00100000:1.0:1788876623.219724:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:1.0:1788876623.220125:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876623.220132:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ff819d25/0x7d14dcdc37047554 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7396 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.220142:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618887ba00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876623.220148:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ff819d25/0x7d14dcdc37047554 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7396 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.220197:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ff819d25/0x7d14dcdc37047554 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7396 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.220206:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876623.220228:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ff819d25/0x7d14dcdc37047554 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7396 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.220236:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ff819d25/0x7d14dcdc37047554 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7396 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.220276:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876623.220278:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876623.220280:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.220296:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511177472:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.224418:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.224448:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532237312 00000100:00100000:1.0:1788876623.224457:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532237312 00000100:00100000:1.0:1788876623.224509:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532237696 00000100:00100000:1.0:1788876623.224513:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532237696 00000100:00100000:1.0:1788876623.224535:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 58 00000100:00100000:1.0:1788876623.224544:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:17045:x1875771532237312:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.224565:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e35a8b1a/0x7d14dcdc370474c1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x204:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a720e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.224639:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.224646:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:17045:x1875771532237312:12345-192.168.203.111@tcp:104: Request processed in 103us (4932us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.224653:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 58 00000100:00100000:1.0:1788876623.224670:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 59 00000100:00100000:1.0:1788876623.224676:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43962:x1875771532237696:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.224685:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc370474eb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x205:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a727e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.224714:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.224720:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43962:x1875771532237696:12345-192.168.203.111@tcp:104: Request processed in 44us (316us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.224739:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 59 00000100:00100000:1.0:1788876623.228221:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.228311:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.228344:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.228455:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ece680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511177344:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.228504:0:33373:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876623.228520:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532237824 00000100:00100000:1.0:1788876623.228526:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532237824 00000100:00100000:1.0:1788876623.228570:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532237952 00000100:00100000:1.0:1788876623.228573:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532237952 00000100:00100000:1.0:1788876623.228586:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532238080 00000100:00100000:1.0:1788876623.228588:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532238080 00000100:00100000:1.0:1788876623.228603:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 60 00000100:00100000:1.0:1788876623.228612:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188847100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43958:x1875771532237824:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.231519:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005b53d543/0x7d14dcdc37047515 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x206:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a72ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.231560:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.231566:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188847100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43958:x1875771532237824:12345-192.168.203.111@tcp:104: Request processed in 2954us (3357us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.231573:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 60 00000100:00100000:1.0:1788876623.235694:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.236325:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.236349:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532239360 00000100:00100000:1.0:1788876623.236359:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532239360 00000100:00100000:1.0:1788876623.236428:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532239232 00000100:00100000:1.0:1788876623.236431:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532239232 00000100:00100000:1.0:1788876623.236465:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 65 00000100:00100000:1.0:1788876623.236474:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532239360:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.236496:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d37049f9/0x7d14dcdc370475fc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x20c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7556 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.236564:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.236571:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532239360:12345-192.168.203.111@tcp:104: Request processed in 98us (887us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.236579:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 65 00000100:00100000:1.0:1788876623.236596:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 66 00000100:00100000:1.0:1788876623.236603:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43960:x1875771532239232:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.236613:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000010fef104/0x7d14dcdc3704753f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x207:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a735e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.236648:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.236654:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43960:x1875771532239232:12345-192.168.203.111@tcp:104: Request processed in 51us (331us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.236660:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 66 00000100:00100000:1.0:1788876623.239101:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.239277:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.239301:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532239488 00000100:00100000:1.0:1788876623.239309:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532239488 00000100:00100000:1.0:1788876623.239343:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532239872 00000100:00100000:1.0:1788876623.239346:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532239872 00000100:00100000:1.0:1788876623.239361:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 67 00000100:00100000:1.0:1788876623.239369:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43959:x1875771532239488:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.239471:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009e81b14c/0x7d14dcdc37047569 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x208:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a73ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.239533:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.239540:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43959:x1875771532239488:12345-192.168.203.111@tcp:104: Request processed in 172us (540us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.239548:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 67 00000100:00100000:1.0:1788876623.239565:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 68 00000100:00100000:1.0:1788876623.239571:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:17045:x1875771532239872:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.239581:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009ba3d29a/0x7d14dcdc370475bd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x20a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a74ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.239605:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.239611:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:17045:x1875771532239872:12345-192.168.203.111@tcp:104: Request processed in 40us (335us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.239617:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 68 00000100:00100000:1.0:1788876623.244276:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.244318:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.244350:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.244408:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.247246:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.247317:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511178496:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876623.247433:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.247461:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.247510:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532241280 00000100:00100000:1.0:1788876623.247516:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532241280 00000100:00100000:1.0:1788876623.247551:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532241408 00000100:00100000:1.0:1788876623.247555:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532241408 00000100:00100000:1.0:1788876623.247602:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 70 00000100:00100000:1.0:1788876623.247613:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188847b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:43953:x1875771532241280:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.247627:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000074b6be50/0x7d14dcdc37047650 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x20e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7636 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.247689:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.247697:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188847b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:43953:x1875771532241280:12345-192.168.203.111@tcp:104: Request processed in 85us (456us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.247712:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 70 00000100:00100000:1.0:1788876623.247733:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 72 00000100:00100000:1.0:1788876623.248069:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43955:x1875771532241536:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.248081:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000832baa45/0x7d14dcdc3704767a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x20f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a76a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.250523:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.250538:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43955:x1875771532241536:12345-192.168.203.111@tcp:104: Request processed in 2793us (3077us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.250550:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 72 00000100:00100000:1.0:1788876623.255243:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.255281:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532241792 00000100:00100000:1.0:1788876623.255288:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532241792 00000100:00100000:1.0:1788876623.255432:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 73 00000100:00100000:1.0:1788876623.255443:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532241792:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.255462:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001e37a41b/0x7d14dcdc3704763b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x20d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a75fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.255525:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.255547:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532241792:12345-192.168.203.111@tcp:104: Request processed in 106us (308us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.255554:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 73 00000100:00100000:1.0:1788876623.255595:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.256132:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.256155:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532241920 00000100:00100000:1.0:1788876623.256161:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532241920 00000100:00100000:1.0:1788876623.256186:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532242176 00000100:00100000:1.0:1788876623.256190:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532242176 00000100:00100000:1.0:1788876623.256603:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 75 00000100:00100000:1.0:1788876623.256617:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:10319:x1875771532242176:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.256630:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001dc9cc83/0x7d14dcdc370476ce lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x211:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7786 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.256676:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.256684:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:10319:x1875771532242176:12345-192.168.203.111@tcp:104: Request processed in 70us (554us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.260138:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 75 00000100:00100000:1.0:1788876623.260513:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.260617:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.260987:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.261038:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.261060:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532243328 00000100:00100000:1.0:1788876623.261066:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532243328 00000100:00100000:1.0:1788876623.261097:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 78 00000100:00100000:1.0:1788876623.261109:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8455:x1875771532243328:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.261120:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000000e9392ce/0x7d14dcdc370476f8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a77f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.261169:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.261174:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8455:x1875771532243328:12345-192.168.203.111@tcp:104: Request processed in 69us (137us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.261180:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 78 00000100:00100000:1.0:1788876623.261514:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.261529:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532243968 00000100:00100000:1.0:1788876623.261533:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532243968 00000100:00100000:1.0:1788876623.261555:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 79 00000100:00100000:1.0:1788876623.262405:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8456:x1875771532243968:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.262418:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000714ffe20/0x7d14dcdc370476b9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x210:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a774e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.262447:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1788876623.262453:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8456:x1875771532243968:12345-192.168.203.111@tcp:104: Request processed in 70us (941us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.265023:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 79 00000100:00100000:1.0:1788876623.265096:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.265232:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.265253:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532244096 00000100:00100000:1.0:1788876623.265258:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532244096 00000100:00100000:1.0:1788876623.265288:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532244224 00000100:00100000:1.0:1788876623.265291:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532244224 00000100:00100000:1.0:1788876623.265304:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 80 00000100:00100000:1.0:1788876623.265312:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43962:x1875771532244096:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.266064:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fe37c5d9/0x7d14dcdc3704774c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a78d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.266148:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.266154:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43962:x1875771532244096:12345-192.168.203.111@tcp:104: Request processed in 843us (1059us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.266161:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 80 00000100:00100000:1.0:1788876623.269669:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.269714:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.269811:0:15182:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876623.269829:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532244608 00000100:00100000:1.0:1788876623.269836:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532244608 00000100:00100000:1.0:1788876623.269890:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532244736 00000100:00100000:1.0:1788876623.269893:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532244736 00000100:00100000:1.0:1788876623.269911:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 82 00000100:00100000:1.0:1788876623.269920:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43961:x1875771532244608:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.269941:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000049632d8f/0x7d14dcdc37047776 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7946 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.270015:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.270022:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43961:x1875771532244608:12345-192.168.203.111@tcp:104: Request processed in 103us (363us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.270029:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 82 00000100:00100000:1.0:1788876623.270045:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 83 00000100:00100000:1.0:1788876623.270052:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190eaa850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532244736:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.270061:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c277c73f/0x7d14dcdc3704770d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x212:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a782e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.270132:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.270151:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190eaa850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532244736:12345-192.168.203.111@tcp:104: Request processed in 99us (437us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.270159:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 83 00000100:00100000:1.0:1788876623.270218:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.270284:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.270304:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532245248 00000100:00100000:1.0:1788876623.270311:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532245248 00000100:00100000:1.0:1788876623.271189:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 85 00000100:00100000:1.0:1788876623.271199:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43963:x1875771532245248:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.271208:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008a214feb/0x7d14dcdc37047737 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x213:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a789e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.271246:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.271253:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:43963:x1875771532245248:12345-192.168.203.111@tcp:104: Request processed in 56us (970us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.271260:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 85 00000100:00100000:1.0:1788876623.275777:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.275808:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532245504 00000100:00100000:1.0:1788876623.275817:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532245504 00000100:00100000:1.0:1788876623.275926:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 86 00000100:00100000:1.0:1788876623.275937:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532245504:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.275957:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009bf36839/0x7d14dcdc37046b28 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ca:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5876 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.276035:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.276043:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532245504:12345-192.168.203.111@tcp:104: Request processed in 108us (275us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.276051:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 86 00000100:00100000:1.0:1788876623.283970:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.284020:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.284069:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.284098:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532245376 00000100:00100000:1.0:1788876623.284117:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532245376 00000100:00100000:1.0:1788876623.284158:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532245888 00000100:00100000:1.0:1788876623.284161:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532245888 00000100:00100000:1.0:1788876623.284172:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532246144 00000100:00100000:1.0:1788876623.284174:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532246144 00000100:00100000:1.0:1788876623.284187:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 87 00000100:00100000:1.0:1788876623.284196:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:10319:x1875771532245376:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.284226:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000027ad87f/0x7d14dcdc370477a0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x216:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a79b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.284297:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.284303:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:10319:x1875771532245376:12345-192.168.203.111@tcp:104: Request processed in 109us (344us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.284311:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 87 00000100:00100000:1.0:1788876623.284329:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 88 00000100:00100000:1.0:1788876623.284336:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8455:x1875771532245888:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.284344:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000dc977afc/0x7d14dcdc370477ca lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7a26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.284370:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.284375:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:8455:x1875771532245888:12345-192.168.203.111@tcp:104: Request processed in 41us (356us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.284412:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 88 00000100:00100000:1.0:1788876623.284424:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 89 00000100:00100000:1.0:1788876623.284446:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43954:x1875771532246144:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.284455:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000028394c6c/0x7d14dcdc370477f4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x218:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7a96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.284484:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876623.284489:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188845180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43954:x1875771532246144:12345-192.168.203.111@tcp:104: Request processed in 45us (421us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.284495:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 89 00010000:00010000:3.0:1788876623.286595:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f17c437a/0x7d14dcdc37046c24 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5b16 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.286606:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f17c437a/0x7d14dcdc37046c24 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5b16 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.286617:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.286621:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f17c437a/0x7d14dcdc37046c24 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5b16 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.286628:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f17c437a/0x7d14dcdc37046c24 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5b16 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.286638:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.286649:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x47:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.286654:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.286657:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.286660:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d030 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.286664:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.286667:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d030 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.286676:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x47:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.286709:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.286718:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.286725:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.286734:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.286745:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.286750:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.286752:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048610 00000080:00010000:3.0:1788876623.286756:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048610 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.286759:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.286767:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.286776:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.286784:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-29, flags=0 00000080:00200000:3.0:1788876623.286788:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.286795:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.286802:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.286810:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.286817:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.286821:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.286823:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048610 00000080:00010000:3.0:1788876623.286827:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048610 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.286830:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.286842:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.286851:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.286856:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-29, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.286878:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.286911:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978e00/0x7d14dcdc37047cf5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa529 expref: -99 pid: 33177 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.286922:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x47:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.286949:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978e00/0x7d14dcdc37047cf5 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa529 expref: -99 pid: 33177 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.286957:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.286963:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.286966:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978e00/0x7d14dcdc37047cf5 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x47:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa529 expref: -99 pid: 33177 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.286982:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048610 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ad8b4 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.286999:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.287019:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188960a80 x1875771511183104/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.287037:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188960a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511183104:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.287574:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00010000:00010000:3.0:1788876623.287595:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876623.287602:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.297014:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:3.0:1788876623.297351:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511182592:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876623.297635:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511182848:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876623.298042:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876623.298050:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e3911dc7/0x7d14dcdc3704781e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7b06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.298068:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188934e00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876623.298074:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e3911dc7/0x7d14dcdc3704781e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7b06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.298122:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e3911dc7/0x7d14dcdc3704781e lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7b06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.298131:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876623.298146:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e3911dc7/0x7d14dcdc3704781e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7b06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.298155:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e3911dc7/0x7d14dcdc3704781e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x219:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7b06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1788876623.298178:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1788876623.298190:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511183232:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876623.298246:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876623.298248:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876623.298250:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876623.305643:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.305683:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.305789:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876623.305820:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532247040 00000100:00100000:1.0:1788876623.305829:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532247040 00000100:00100000:1.0:1788876623.305866:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532247168 00000100:00100000:1.0:1788876623.305869:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532247168 00000100:00100000:1.0:1788876623.305878:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532247424 00000100:00100000:1.0:1788876623.305881:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532247424 00000100:00100000:1.0:1788876623.305894:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 91 00000100:00100000:1.0:1788876623.305903:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:17045:x1875771532247040:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.305923:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d92a8621/0x7d14dcdc37047761 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a790e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.306020:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876623.306028:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188846300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:17045:x1875771532247040:12345-192.168.203.111@tcp:104: Request processed in 125us (395us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.306034:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 91 00000100:00100000:1.0:1788876623.306053:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 92 00000100:00100000:1.0:1788876623.306082:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8453:x1875771532247168:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.306091:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007df0f3d4/0x7d14dcdc3704778b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a797e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.306120:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876623.306125:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8453:x1875771532247168:12345-192.168.203.111@tcp:104: Request processed in 46us (443us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.306130:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 92 00000100:00100000:1.0:1788876623.306142:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 93 00000100:00100000:1.0:1788876623.306148:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188844700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43959:x1875771532247424:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876623.306156:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054b99255/0x7d14dcdc370477df lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7a5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876623.306180:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876623.306186:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188844700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43959:x1875771532247424:12345-192.168.203.111@tcp:104: Request processed in 38us (398us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876623.306191:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 93 00000100:00100000:3.0:1788876623.306498:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188960a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511183104:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.306554:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x47:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.306566:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.306594:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x47:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.306638:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x47:0x0](ffffa0619817db88) 00000008:00010000:3.0:1788876623.306671:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f17c437a/0x7d14dcdc37046c24 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1d0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5b16 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.306747:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.306765:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.306816:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.306825:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.306836:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.306854:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188960000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511183360:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.306899:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00010000:00010000:3.0:1788876623.306917:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1788876623.306924:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d92a8621/0x7d14dcdc37047761 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a790e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.306937:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061888f9200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876623.306942:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d92a8621/0x7d14dcdc37047761 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a790e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.306972:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d92a8621/0x7d14dcdc37047761 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a790e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.306979:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876623.306998:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d92a8621/0x7d14dcdc37047761 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a790e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307016:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d92a8621/0x7d14dcdc37047761 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x214:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a790e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307033:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1788876623.307037:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007df0f3d4/0x7d14dcdc3704778b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a797e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307043:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188780e00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876623.307047:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007df0f3d4/0x7d14dcdc3704778b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a797e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307066:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007df0f3d4/0x7d14dcdc3704778b lrc: 2/0,0 mode: --/PW res: [0x280000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a797e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307080:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876623.307089:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007df0f3d4/0x7d14dcdc3704778b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a797e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307096:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007df0f3d4/0x7d14dcdc3704778b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x215:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a797e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307109:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876623.307113:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054b99255/0x7d14dcdc370477df lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7a5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307119:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061887ac800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876623.307123:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054b99255/0x7d14dcdc370477df lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7a5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307143:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054b99255/0x7d14dcdc370477df lrc: 2/0,0 mode: --/PW res: [0x280000401:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7a5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307149:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876623.307154:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054b99255/0x7d14dcdc370477df lrc: 1/0,0 mode: --/PW res: [0x280000401:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7a5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307160:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054b99255/0x7d14dcdc370477df lrc: 0/0,0 mode: --/PW res: [0x280000401:0x217:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7a5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876623.307175:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876623.307177:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876623.307179:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876623.307192:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511183488:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876623.308017:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511183744:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876623.312364:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511183232:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876623.313984:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188960000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511183360:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.314014:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314027:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314040:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314052:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314130:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314140:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314147:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.314164:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188960000 x1875771511183360/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.314179:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314193:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.314200:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.314290:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.314297:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048617 00000080:00010000:3.0:1788876623.314313:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.314317:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048617 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.314320:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.314330:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048617 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ad91d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.314351:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.314361:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-3, flags=8 00000080:00200008:3.0:1788876623.314373:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-3:[0x200001b71:0x2d:0x0](ffffa06198220908), request_mask 2047, flags 0, foreign 0 00000100:00100000:3.0:1788876623.322766:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188960700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511183488:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876623.322776:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511183744:192.168.203.111@tcp:103:ldlm_bl.0 00000080:00200000:3.0:1788876623.427071:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x4d:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788876623.427084:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.427125:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x4d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.427171:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x4d:0x0](ffffa06198228908) 00000008:00010000:3.0:1788876623.427204:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003bbf85ef/0x7d14dcdc37046ca2 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5c66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.427278:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.427298:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.427361:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.427370:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.427405:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.427428:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511186816:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.427500:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.435257:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511186816:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.435286:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435298:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435312:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435328:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435342:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435351:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435359:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.435493:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188961c00 x1875771511186816/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.435510:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435527:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.435533:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.435560:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.435565:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048648 00000080:00010000:3.0:1788876623.435571:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.435574:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048648 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.435577:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.435592:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.435614:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.435624:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-36, flags=8 00000080:00200008:3.0:1788876623.435635:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-36:[0x200001b71:0x4e:0x0](ffffa0619822c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.435640:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x4e:0x0](ffffa0619822c2c8),name=f5.replay-single-36 00000002:00010000:3.0:1788876623.435657:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x4e:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.435667:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ace00/0x7d14dcdc37047d26 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa61e expref: -99 pid: 33184 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.435678:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ace00/0x7d14dcdc37047d26 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa61e expref: -99 pid: 33184 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.435684:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.435687:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x4e:0x0](ffffa0619822c2c8) 00000080:00010000:3.0:1788876623.435689:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x4e:0x0](ffffa0619822c2c8) for lock 0x7d14dcdc37047d26 00000080:00010000:3.0:1788876623.435693:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d26 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.435695:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ace00/0x7d14dcdc37047d26 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa61e expref: -99 pid: 33184 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.435702:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ace00/0x7d14dcdc37047d26 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa61e expref: -99 pid: 33184 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.435720:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x4e:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.435730:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x4e:0x0] 00000008:00010000:3.0:1788876623.435748:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.435752:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d330, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.435768:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.435779:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.435790:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.435794:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.435802:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.435810:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.435820:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x4e:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.435823:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.435826:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.435829:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d330 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.435832:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.435835:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d330 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.435843:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x4e:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.435871:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.435877:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.435883:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.435896:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.435902:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.435905:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.435907:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048648 00000080:00010000:3.0:1788876623.435910:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048648 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.435913:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.435920:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.435928:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.435935:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-36, flags=0 00000080:00200000:3.0:1788876623.435944:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.435950:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.435955:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.435963:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.435970:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.435974:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.435976:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048648 00000080:00010000:3.0:1788876623.435979:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048648 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.435983:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.435989:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.435998:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.436013:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-36, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.436187:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.436222:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ace00/0x7d14dcdc37047d26 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa61e expref: -99 pid: 33184 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.436232:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x4e:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.436265:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ace00/0x7d14dcdc37047d26 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa61e expref: -99 pid: 33184 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.436272:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.436278:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.436281:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889ace00/0x7d14dcdc37047d26 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x4e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa61e expref: -99 pid: 33184 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.436297:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048648 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ada43 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.436602:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.436611:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188961c00 x1875771511187072/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.436633:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187072:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.436722:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.442069:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187072:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.442201:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x4e:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.442216:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.442246:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x4e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.442292:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x4e:0x0](ffffa0619822c2c8) 00000008:00010000:1.0:1788876623.442328:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.442773:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.442796:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.442870:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.442880:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.442890:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.442910:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187200:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.443038:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64240 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.445923:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187200:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.445954:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.445967:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.445982:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.445994:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.446017:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.446027:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.446035:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.446057:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188844380 x1875771511187200/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.446071:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.446085:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.446092:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.446117:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.446124:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704864f 00000080:00010000:3.0:1788876623.446129:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.446132:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704864f from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.446136:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.446145:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.446926:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.446940:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-37, flags=8 00000080:00200008:3.0:1788876623.446952:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-37:[0x200001b71:0x4f:0x0](ffffa0619817ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.446957:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x4f:0x0](ffffa0619817ba88),name=f5.replay-single-37 00000002:00010000:3.0:1788876623.446979:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x4f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.446991:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890bc00/0x7d14dcdc37047d2d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa641 expref: -99 pid: 33185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.447113:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890bc00/0x7d14dcdc37047d2d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa641 expref: -99 pid: 33185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.447124:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.447130:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x4f:0x0](ffffa0619817ba88) 00000080:00010000:3.0:1788876623.447133:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x4f:0x0](ffffa0619817ba88) for lock 0x7d14dcdc37047d2d 00000080:00010000:3.0:1788876623.447138:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d2d from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.447141:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890bc00/0x7d14dcdc37047d2d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa641 expref: -99 pid: 33185 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.447150:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890bc00/0x7d14dcdc37047d2d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa641 expref: -99 pid: 33185 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.447171:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x4f:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:3.0:1788876623.447180:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x4f:0x0] 00000008:00010000:3.0:1788876623.447199:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.447202:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d030, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.447218:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.447230:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.447244:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.447360:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.447370:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.447738:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.447752:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x4f:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.447756:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.447760:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.447764:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d030 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.447768:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.447770:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d030 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.447781:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x4f:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.447814:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.447823:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.447833:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.447908:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.447917:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.447922:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.447925:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704864f 00000080:00010000:3.0:1788876623.447929:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704864f from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.447933:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.447941:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.447957:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.447964:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-37, flags=0 00000080:00200000:3.0:1788876623.447968:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.447975:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.447982:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.447991:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.447999:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.448012:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.448015:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704864f 00000080:00010000:3.0:1788876623.448019:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704864f from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.448023:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.448031:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.448040:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.448046:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-37, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.448070:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.448106:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890bc00/0x7d14dcdc37047d2d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa641 expref: -99 pid: 33185 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.448121:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x4f:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.448158:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890bc00/0x7d14dcdc37047d2d lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa641 expref: -99 pid: 33185 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.448166:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.448173:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.448176:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890bc00/0x7d14dcdc37047d2d lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x4f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa641 expref: -99 pid: 33185 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.448194:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704864f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ada6d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.448212:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.448222:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188960700 x1875771511187456/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.448245:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188960700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187456:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.448368:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.452371:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188960700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187456:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.452666:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x4f:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.452679:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.452705:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x4f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.452764:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x4f:0x0](ffffa0619817ba88) 00000008:00010000:3.0:1788876623.452801:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.452897:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.452918:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.452980:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.452989:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.452999:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.453031:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188960000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187584:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.453151:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.470970:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188960000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187584:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.471009:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471024:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471038:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471050:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471069:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471078:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471086:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.471098:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188960000 x1875771511187584/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.471112:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471125:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.471132:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.471159:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.471166:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048656 00000080:00010000:3.0:1788876623.471172:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.471175:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048656 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.471179:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.471189:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.471218:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.471229:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-38, flags=8 00000080:00200008:3.0:1788876623.471240:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-38:[0x200001b71:0x50:0x0](ffffa0619817ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.471247:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x50:0x0](ffffa0619817ec08),name=f5.replay-single-38 00000002:00010000:3.0:1788876623.471267:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x50:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.471278:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889aca00/0x7d14dcdc37047d34 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa664 expref: -99 pid: 33186 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.471290:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889aca00/0x7d14dcdc37047d34 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa664 expref: -99 pid: 33186 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.471296:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.471300:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x50:0x0](ffffa0619817ec08) 00000080:00010000:3.0:1788876623.471303:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x50:0x0](ffffa0619817ec08) for lock 0x7d14dcdc37047d34 00000080:00010000:3.0:1788876623.471307:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d34 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.471310:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889aca00/0x7d14dcdc37047d34 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa664 expref: -99 pid: 33186 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.471318:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889aca00/0x7d14dcdc37047d34 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa664 expref: -99 pid: 33186 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.471337:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x50:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.471347:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x50:0x0] 00000008:00010000:3.0:1788876623.471371:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.471375:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d330, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.471459:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.471472:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.471485:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.471488:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.471496:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.471505:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.471516:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x50:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.471520:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.471523:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.471526:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d330 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.471530:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.471533:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d330 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.471542:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x50:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.471573:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.471581:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.471588:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.471596:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.471607:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.471612:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.471615:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048656 00000080:00010000:3.0:1788876623.471618:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048656 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.471621:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.471628:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.471637:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.471643:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-38, flags=0 00000080:00200000:3.0:1788876623.471648:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.471654:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.471661:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.471669:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.471675:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.471679:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.471682:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048656 00000080:00010000:3.0:1788876623.471686:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048656 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.471689:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.471701:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.471710:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.471715:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-38, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.471738:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.471772:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889aca00/0x7d14dcdc37047d34 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa664 expref: -99 pid: 33186 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.471784:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x50:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.471816:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889aca00/0x7d14dcdc37047d34 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa664 expref: -99 pid: 33186 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.471825:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.471831:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.471834:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061889aca00/0x7d14dcdc37047d34 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x50:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa664 expref: -99 pid: 33186 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.471851:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048656 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ada97 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.471869:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.471877:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188960000 x1875771511187840/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.471898:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188960000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187840:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.471978:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.476020:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188960000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187840:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.476087:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x50:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.476103:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.476136:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x50:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.476191:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x50:0x0](ffffa0619817ec08) 00000008:00010000:3.0:1788876623.476228:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.476322:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.476344:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.476458:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.476468:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.476477:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.476499:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187968:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.476578:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.490285:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511187968:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.490317:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.490331:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.490345:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.490357:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.490370:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.490380:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.490430:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.490442:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188963100 x1875771511187968/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.490457:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ae00/0x7d14dcdc3704865d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adac1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.490475:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.490481:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000100:00100000:1.0:1788876623.528659:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecea00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511188992:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.528736:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x52:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788876623.528751:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.528779:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x52:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.528839:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x52:0x0](ffffa0619822cb08) 00000008:00010000:1.0:1788876623.528882:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d3549a53/0x7d14dcdc37046d0b lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1d5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5d7e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.528973:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.528994:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.529323:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.529334:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.529346:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.529469:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189120:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.529550:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.532173:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189120:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.532200:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532213:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532227:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532239:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532253:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532262:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532270:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.532283:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188845f80 x1875771511189120/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.532315:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532330:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.532336:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.532361:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.532367:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048672 00000080:00010000:1.0:1788876623.532372:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.532375:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048672 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.532378:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.532417:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.532443:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.532454:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-41, flags=8 00000080:00200008:1.0:1788876623.532465:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-41:[0x200001b71:0x53:0x0](ffffa0619822f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.532471:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x53:0x0](ffffa0619822f448),name=f5.replay-single-41 00000002:00010000:1.0:1788876623.532491:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x53:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.532500:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b200/0x7d14dcdc37047d49 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa6cd expref: -99 pid: 33190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.532511:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b200/0x7d14dcdc37047d49 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa6cd expref: -99 pid: 33190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.532518:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.532529:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x53:0x0](ffffa0619822f448) 00000080:00010000:1.0:1788876623.532532:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x53:0x0](ffffa0619822f448) for lock 0x7d14dcdc37047d49 00000080:00010000:1.0:1788876623.532536:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d49 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.532540:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b200/0x7d14dcdc37047d49 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa6cd expref: -99 pid: 33190 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.532547:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b200/0x7d14dcdc37047d49 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa6cd expref: -99 pid: 33190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.532571:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x53:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.532582:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x53:0x0] 00000008:00010000:1.0:1788876623.532604:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.532607:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.532628:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.532641:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.532653:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.532656:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.532664:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.532674:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.532686:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x53:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.532696:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.532699:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.532702:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.532707:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.532709:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.532720:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x53:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.532752:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.532760:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.532769:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.532778:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.532785:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.532789:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.532792:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048672 00000080:00010000:1.0:1788876623.532796:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048672 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.532800:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.532807:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.532817:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.532824:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-41, flags=0 00000080:00200000:1.0:1788876623.532828:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.532834:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.532859:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.532869:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.532876:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.532881:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.532884:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048672 00000080:00010000:1.0:1788876623.532888:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048672 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.532891:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.532899:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.532908:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.532914:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-41, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.532938:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.532972:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b200/0x7d14dcdc37047d49 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa6cd expref: -99 pid: 33190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.532982:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x53:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.533027:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b200/0x7d14dcdc37047d49 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa6cd expref: -99 pid: 33190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.533035:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.533050:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.533052:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b200/0x7d14dcdc37047d49 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x53:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa6cd expref: -99 pid: 33190 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.533084:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048672 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adb3f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.533102:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.533110:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188845f80 x1875771511189376/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.533132:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189376:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.533194:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.539120:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189376:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.539182:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x53:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.539194:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.539219:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x53:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.539263:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x53:0x0](ffffa0619822f448) 00000008:00010000:1.0:1788876623.539366:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.539516:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.539534:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.539590:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.539613:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.539624:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.539642:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189504:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.539695:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.542937:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189504:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.542974:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.542988:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.543009:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.543021:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.543036:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.543044:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.543083:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.543098:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188845180 x1875771511189504/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.543112:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.543127:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.543134:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.543162:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.543169:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048679 00000080:00010000:1.0:1788876623.543174:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.543177:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048679 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.543181:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.543189:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.543214:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.543226:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-42, flags=8 00000080:00200008:1.0:1788876623.543237:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-42:[0x200001b71:0x54:0x0](ffffa0619822b248), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.543243:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x54:0x0](ffffa0619822b248),name=f5.replay-single-42 00000002:00010000:1.0:1788876623.543261:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x54:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.543272:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a200/0x7d14dcdc37047d50 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa6f0 expref: -99 pid: 33191 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.543290:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a200/0x7d14dcdc37047d50 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa6f0 expref: -99 pid: 33191 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.543297:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.543302:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x54:0x0](ffffa0619822b248) 00000080:00010000:1.0:1788876623.543305:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x54:0x0](ffffa0619822b248) for lock 0x7d14dcdc37047d50 00000080:00010000:1.0:1788876623.543309:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d50 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.543312:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a200/0x7d14dcdc37047d50 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa6f0 expref: -99 pid: 33191 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.543320:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a200/0x7d14dcdc37047d50 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa6f0 expref: -99 pid: 33191 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.543340:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x54:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.543350:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x54:0x0] 00000008:00010000:1.0:1788876623.543371:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.543374:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.543424:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.543438:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.543450:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.543454:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.543474:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.543485:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.543498:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x54:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.543503:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.543506:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.543510:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.543513:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.543516:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.543526:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x54:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.543558:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.543567:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.543574:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.543583:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.543589:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.543594:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.543597:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048679 00000080:00010000:1.0:1788876623.543600:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048679 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.543604:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.543622:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.543631:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.543639:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-42, flags=0 00000080:00200000:1.0:1788876623.543643:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.543650:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.543656:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.543665:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.543671:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.543675:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.543678:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048679 00000080:00010000:1.0:1788876623.543682:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048679 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.543685:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.543693:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.543702:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.543707:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-42, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.543729:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.543769:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a200/0x7d14dcdc37047d50 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa6f0 expref: -99 pid: 33191 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.543780:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x54:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.543813:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a200/0x7d14dcdc37047d50 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa6f0 expref: -99 pid: 33191 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.543821:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.543828:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.543831:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a200/0x7d14dcdc37047d50 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x54:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa6f0 expref: -99 pid: 33191 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.543850:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048679 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adb69 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.543866:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.543874:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188845180 x1875771511189760/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.543894:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189760:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.543955:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.551177:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189760:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.551242:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x54:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.551256:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.551287:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x54:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.551356:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x54:0x0](ffffa0619822b248) 00000008:00010000:1.0:1788876623.551410:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876614 ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.551489:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.551509:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.551569:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.551578:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.551588:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.551608:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847b80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189888:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.551686:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.563206:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188847b80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511189888:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.563236:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563249:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563263:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563274:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563305:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563313:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563322:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.563333:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188847b80 x1875771511189888/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.563347:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563361:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.563367:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.563541:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.563548:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048680 00000080:00010000:1.0:1788876623.563554:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.563557:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048680 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.563561:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.563571:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.563606:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.563616:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-43, flags=8 00000080:00200008:1.0:1788876623.563627:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-43:[0x200001b71:0x55:0x0](ffffa0619822ec08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.563633:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x55:0x0](ffffa0619822ec08),name=f5.replay-single-43 00000002:00010000:1.0:1788876623.563654:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x55:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.563665:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978200/0x7d14dcdc37047d57 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa713 expref: -99 pid: 33192 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.563676:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978200/0x7d14dcdc37047d57 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa713 expref: -99 pid: 33192 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.563683:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.563688:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x55:0x0](ffffa0619822ec08) 00000080:00010000:1.0:1788876623.563691:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x55:0x0](ffffa0619822ec08) for lock 0x7d14dcdc37047d57 00000080:00010000:1.0:1788876623.563694:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d57 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.563698:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978200/0x7d14dcdc37047d57 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa713 expref: -99 pid: 33192 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.563705:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978200/0x7d14dcdc37047d57 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa713 expref: -99 pid: 33192 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.563789:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x55:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.563801:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x55:0x0] 00000008:00010000:1.0:1788876623.563820:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.563824:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.564169:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.564189:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.564202:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.564205:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.564213:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.564224:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.564235:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x55:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.564239:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.564242:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.564246:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.564250:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.564253:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.564263:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x55:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.564296:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.564305:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.564313:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.564323:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.564340:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.564344:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.564347:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048680 00000080:00010000:1.0:1788876623.564352:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048680 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.564355:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.564362:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.564372:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.564379:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-43, flags=0 00000080:00200000:1.0:1788876623.564410:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.564419:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.564426:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.564434:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.564440:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.564444:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.564447:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048680 00000080:00010000:1.0:1788876623.564450:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048680 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.564453:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.564471:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.564480:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.564486:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-43, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.564508:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.564539:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978200/0x7d14dcdc37047d57 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa713 expref: -99 pid: 33192 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.564550:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x55:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.564581:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978200/0x7d14dcdc37047d57 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa713 expref: -99 pid: 33192 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.564589:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.564595:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.564599:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978200/0x7d14dcdc37047d57 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x55:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa713 expref: -99 pid: 33192 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.564617:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048680 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adb93 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.564633:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.564643:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188847b80 x1875771511190144/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.564670:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847b80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190144:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.573179:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.578754:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188847b80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190144:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.578816:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x55:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.578828:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.578866:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x55:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.578913:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x55:0x0](ffffa0619822ec08) 00000008:00010000:1.0:1788876623.578952:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.579046:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.579067:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.579128:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.579136:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.579146:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.579165:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190272:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.579239:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.586367:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188847480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190272:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.586432:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586463:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586476:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586488:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586501:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586509:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586515:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.586526:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188847480 x1875771511190272/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.586540:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586554:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.586561:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.586587:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.586600:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048687 00000080:00010000:1.0:1788876623.586605:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.586608:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048687 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.586612:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.586621:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.586645:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.586656:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-44, flags=8 00000080:00200008:1.0:1788876623.586667:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-44:[0x200001b71:0x56:0x0](ffffa0619822ba88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.586673:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x56:0x0](ffffa0619822ba88),name=f5.replay-single-44 00000002:00010000:1.0:1788876623.586696:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x56:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.586707:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978400/0x7d14dcdc37047d5e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa736 expref: -99 pid: 33193 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.586733:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978400/0x7d14dcdc37047d5e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa736 expref: -99 pid: 33193 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.586740:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.586744:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x56:0x0](ffffa0619822ba88) 00000080:00010000:1.0:1788876623.586747:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x56:0x0](ffffa0619822ba88) for lock 0x7d14dcdc37047d5e 00000080:00010000:1.0:1788876623.586751:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d5e from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.586754:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978400/0x7d14dcdc37047d5e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa736 expref: -99 pid: 33193 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.586761:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978400/0x7d14dcdc37047d5e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa736 expref: -99 pid: 33193 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.586787:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x56:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.586796:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x56:0x0] 00000008:00010000:1.0:1788876623.586816:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.586819:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.586836:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37046d5f lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.586856:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37046d5f lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.586868:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.586871:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37046d5f lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5e5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.586879:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37046d5f lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.586889:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.586900:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x56:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.586904:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.586907:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.586910:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.586914:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.586917:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.586928:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x56:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.586958:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.586972:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.586979:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.586987:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.586993:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.586997:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.587009:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048687 00000080:00010000:1.0:1788876623.587013:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048687 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.587017:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.587024:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.587032:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.587039:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-44, flags=0 00000080:00200000:1.0:1788876623.587043:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.587049:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.587055:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.587063:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.587070:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.587073:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.587081:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048687 00000080:00010000:1.0:1788876623.587085:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048687 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.587089:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.587096:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.587105:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.587110:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-44, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.587135:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.587166:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978400/0x7d14dcdc37047d5e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa736 expref: -99 pid: 33193 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.587177:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x56:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.587208:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978400/0x7d14dcdc37047d5e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa736 expref: -99 pid: 33193 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.587216:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.587222:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.587225:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978400/0x7d14dcdc37047d5e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x56:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa736 expref: -99 pid: 33193 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.587243:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048687 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adbbd expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.587271:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.587279:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188847480 x1875771511190528/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.587300:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190528:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.587366:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.598263:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188847480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190528:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.598346:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x56:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.598377:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.598462:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x56:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.598521:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x56:0x0](ffffa0619822ba88) 00000008:00010000:1.0:1788876623.598561:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37046d5f lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1d7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e5e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.598640:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.598660:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.602253:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.602270:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.602287:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.602320:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190656:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.602440:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.605085:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190656:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.605111:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605124:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605138:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605149:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605164:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605171:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605178:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.605190:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188846d80 x1875771511190656/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.605203:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605299:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.605306:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.605331:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.605336:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704868e 00000080:00010000:1.0:1788876623.605341:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.605344:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704868e from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.605358:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.605369:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.605430:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.605443:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-45, flags=8 00000080:00200008:1.0:1788876623.605456:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-45:[0x200001b71:0x57:0x0](ffffa0619817f448), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.605463:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x57:0x0](ffffa0619817f448),name=f5.replay-single-45 00000002:00010000:1.0:1788876623.605485:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x57:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.605497:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188979000/0x7d14dcdc37047d65 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x57:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa759 expref: -99 pid: 33194 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.605508:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188979000/0x7d14dcdc37047d65 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x57:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa759 expref: -99 pid: 33194 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.605515:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.605519:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x57:0x0](ffffa0619817f448) 00000080:00010000:1.0:1788876623.605521:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x57:0x0](ffffa0619817f448) for lock 0x7d14dcdc37047d65 00000080:00010000:1.0:1788876623.605532:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d65 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.605535:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188979000/0x7d14dcdc37047d65 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x57:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa759 expref: -99 pid: 33194 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.605542:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188979000/0x7d14dcdc37047d65 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x57:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa759 expref: -99 pid: 33194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.605562:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x57:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.605572:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x57:0x0] 00000008:00010000:1.0:1788876623.605593:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.605596:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.605612:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.605624:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.605636:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.605639:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.605646:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.605656:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.605668:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x57:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.605671:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.605674:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.605684:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.605687:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.605690:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.605701:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x57:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.605730:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.605737:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.605744:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.605752:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.605759:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.605763:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.605765:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704868e 00000080:00010000:1.0:1788876623.605768:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704868e from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.605771:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.605778:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.605786:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.605793:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-45, flags=0 00000080:00200000:1.0:1788876623.605797:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.605802:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.605807:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.605826:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.605833:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.605836:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.605838:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704868e 00000080:00010000:1.0:1788876623.605843:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704868e from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.605845:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.605852:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.605859:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.605865:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-45, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.605888:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.605919:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188979000/0x7d14dcdc37047d65 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x57:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa759 expref: -99 pid: 33194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.605929:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x57:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.605962:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188979000/0x7d14dcdc37047d65 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x57:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa759 expref: -99 pid: 33194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.605969:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.605985:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.605989:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188979000/0x7d14dcdc37047d65 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x57:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa759 expref: -99 pid: 33194 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.606018:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704868e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adbe7 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.606038:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.606046:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188846d80 x1875771511190912/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.606068:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190912:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.606132:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.616876:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511190912:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.616960:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x57:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.616974:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.617013:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x57:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.617069:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x57:0x0](ffffa0619817f448) 00000008:00010000:1.0:1788876623.617106:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.617188:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.617210:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.617269:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.617292:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.617303:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.617322:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511191040:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.617576:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.631634:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511191040:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.631664:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631677:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631691:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631702:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631716:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631723:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631750:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.631762:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188845c00 x1875771511191040/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.631776:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631790:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.631796:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.631821:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.631827:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048695 00000080:00010000:1.0:1788876623.631832:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.631836:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048695 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.631848:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.631858:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.631883:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.631894:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-46, flags=8 00000080:00200008:1.0:1788876623.631905:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-46:[0x200001b71:0x58:0x0](ffffa061982280c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.632091:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x58:0x0](ffffa061982280c8),name=f5.replay-single-46 00000002:00010000:1.0:1788876623.632114:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x58:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.632126:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978a00/0x7d14dcdc37047d6c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x58:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa77c expref: -99 pid: 33195 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.632148:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978a00/0x7d14dcdc37047d6c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x58:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa77c expref: -99 pid: 33195 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.632156:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.632162:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x58:0x0](ffffa061982280c8) 00000080:00010000:1.0:1788876623.632164:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x58:0x0](ffffa061982280c8) for lock 0x7d14dcdc37047d6c 00000080:00010000:1.0:1788876623.632169:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d6c from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.632172:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978a00/0x7d14dcdc37047d6c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x58:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa77c expref: -99 pid: 33195 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.632180:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978a00/0x7d14dcdc37047d6c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x58:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa77c expref: -99 pid: 33195 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.632201:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x58:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.632212:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x58:0x0] 00000008:00010000:1.0:1788876623.632231:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.632235:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.632253:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002b348e81/0x7d14dcdc37046d89 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d8:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5ece expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.632264:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002b348e81/0x7d14dcdc37046d89 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5ece expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.632276:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.632279:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002b348e81/0x7d14dcdc37046d89 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5ece expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.632298:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002b348e81/0x7d14dcdc37046d89 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5ece expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.632309:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.632320:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x58:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.632324:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.632327:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.632330:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.632333:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.632336:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.632347:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x58:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.632469:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.632477:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.632485:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.632493:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.632499:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.632503:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.632506:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048695 00000080:00010000:1.0:1788876623.632509:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048695 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.632512:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.632527:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.632535:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.632543:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-46, flags=0 00000080:00200000:1.0:1788876623.632547:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.632554:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.632561:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.632569:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.632575:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.632579:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.632581:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048695 00000080:00010000:1.0:1788876623.632585:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048695 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.632588:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.632595:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.632602:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.632607:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-46, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.632629:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.632667:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978a00/0x7d14dcdc37047d6c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x58:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa77c expref: -99 pid: 33195 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.632679:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x58:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.632715:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978a00/0x7d14dcdc37047d6c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x58:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa77c expref: -99 pid: 33195 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.632723:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.632730:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.632733:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188978a00/0x7d14dcdc37047d6c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x58:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa77c expref: -99 pid: 33195 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.632751:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048695 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adc11 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.632768:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.632776:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188845c00 x1875771511191296/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.632797:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511191296:192.168.203.111@tcp:36:rm.0 00010000:00010000:3.0:1788876623.661719:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.661742:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.661755:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.661759:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486a3 00000080:00010000:3.0:1788876623.661770:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486a3 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.661774:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.661784:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.661800:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.661816:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-48, flags=0 00000080:00200000:3.0:1788876623.661823:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.661841:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.661856:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.661865:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.661872:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.661877:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.661879:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486a3 00000080:00010000:3.0:1788876623.661883:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486a3 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.661886:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.661898:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.661906:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.661911:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-48, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.661940:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.661972:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b400/0x7d14dcdc37047d7a lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x5a:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa7c2 expref: -99 pid: 33197 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.661983:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x5a:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.662035:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b400/0x7d14dcdc37047d7a lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x5a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa7c2 expref: -99 pid: 33197 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.662043:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.662049:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.662052:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b400/0x7d14dcdc37047d7a lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x5a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa7c2 expref: -99 pid: 33197 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.662070:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031000/0x7d14dcdc370486a3 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adc65 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.662091:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.662103:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188960e00 x1875771511192064/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.662129:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188960e00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192064:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.662206:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.667895:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188960e00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192064:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.667958:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x5a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788876623.667971:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.668010:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x5a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.668061:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x5a:0x0](ffffa061982342c8) 00000008:00010000:3.0:1788876623.668099:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f7fc4299/0x7d14dcdc37046db3 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1d9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5f3e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.668179:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.668199:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.668256:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.668266:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.668276:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.668295:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192192:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.668373:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.676254:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192192:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.676283:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676296:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676310:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676321:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676334:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676343:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676366:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.676378:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188962680 x1875771511192192/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.676672:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676688:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.676695:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.676721:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.676728:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486aa 00000080:00010000:3.0:1788876623.676734:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.676737:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486aa from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.676741:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.676751:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.676775:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.676787:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-49, flags=8 00000080:00200008:3.0:1788876623.676799:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-49:[0x200001b71:0x5b:0x0](ffffa06198234b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.676805:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x5b:0x0](ffffa06198234b08),name=f5.replay-single-49 00000002:00010000:3.0:1788876623.676825:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x5b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.676836:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a800/0x7d14dcdc37047d81 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa7e5 expref: -99 pid: 33198 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.676918:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a800/0x7d14dcdc37047d81 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa7e5 expref: -99 pid: 33198 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.676927:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.676931:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x5b:0x0](ffffa06198234b08) 00000080:00010000:3.0:1788876623.676935:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x5b:0x0](ffffa06198234b08) for lock 0x7d14dcdc37047d81 00000080:00010000:3.0:1788876623.676939:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047d81 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.676943:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a800/0x7d14dcdc37047d81 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa7e5 expref: -99 pid: 33198 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.676951:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a800/0x7d14dcdc37047d81 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa7e5 expref: -99 pid: 33198 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.676972:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x5b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.676983:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x5b:0x0] 00000008:00010000:3.0:1788876623.677011:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.677015:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d6b0, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.677032:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.677044:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.677056:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.677060:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.677068:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.677078:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.677090:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x5b:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.677093:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.677097:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.677100:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d6b0 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.677104:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.677107:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d6b0 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.677117:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x5b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.677149:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.677157:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.677165:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.677174:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.677180:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.677184:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.677187:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486aa 00000080:00010000:3.0:1788876623.677191:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486aa from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.677194:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.677202:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.677211:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.677218:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-49, flags=0 00000080:00200000:3.0:1788876623.677222:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.677228:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.677234:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.677242:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.677249:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.677252:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.677255:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486aa 00000080:00010000:3.0:1788876623.677259:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486aa from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.677263:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.677269:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.677278:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.677283:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-49, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.677306:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.677338:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a800/0x7d14dcdc37047d81 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa7e5 expref: -99 pid: 33198 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.677362:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x5b:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.678746:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a800/0x7d14dcdc37047d81 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa7e5 expref: -99 pid: 33198 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.678762:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.678768:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.678772:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890a800/0x7d14dcdc37047d81 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x5b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa7e5 expref: -99 pid: 33198 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.678791:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486aa lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adc8f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.678813:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.678824:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188962680 x1875771511192448/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.680720:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192448:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.680812:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.686168:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192448:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.686228:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x5b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.686241:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.686267:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x5b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.686317:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x5b:0x0](ffffa06198234b08) 00000008:00010000:3.0:1788876623.686368:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876559 1788876559 0 by oinfo size 7 blocks 8 [cma]time 1788876559 1788876559 1788876615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.686465:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.686484:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.686542:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.686552:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.686563:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.686717:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962a00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192576:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.686780:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.689130:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962a00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511192576:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.689158:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689172:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689187:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689200:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689214:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689224:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689232:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.689243:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188962a00 x1875771511192576/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.689256:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689277:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.689284:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.689310:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.689316:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486b1 00000080:00010000:3.0:1788876623.689322:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.689326:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486b1 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.689329:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.689340:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a400/0x7d14dcdc370486b1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adcb9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.689378:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.689693:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-5, flags=8 00000080:00200000:3.0:1788876623.756897:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.756921:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.756985:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.756995:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.757015:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.757038:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962a00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511194496:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.757119:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.760031:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962a00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511194496:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.760057:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760069:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760081:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760093:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760107:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760115:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760123:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.760135:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188962a00 x1875771511194496/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.760149:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760163:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.760170:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.760199:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.760205:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486d4 00000080:00010000:3.0:1788876623.760210:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.760213:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486d4 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.760217:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.760226:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.760249:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.760259:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-54, flags=8 00000080:00200008:3.0:1788876623.760270:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-54:[0x200001b71:0x60:0x0](ffffa061982300c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.760275:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x60:0x0](ffffa061982300c8),name=f5.replay-single-54 00000002:00010000:3.0:1788876623.760295:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x60:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.760305:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b800/0x7d14dcdc37047da4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa894 expref: -99 pid: 33203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.760317:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b800/0x7d14dcdc37047da4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa894 expref: -99 pid: 33203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.760324:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.760328:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x60:0x0](ffffa061982300c8) 00000080:00010000:3.0:1788876623.760331:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x60:0x0](ffffa061982300c8) for lock 0x7d14dcdc37047da4 00000080:00010000:3.0:1788876623.760335:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047da4 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.760339:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b800/0x7d14dcdc37047da4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa894 expref: -99 pid: 33203 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.760363:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b800/0x7d14dcdc37047da4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa894 expref: -99 pid: 33203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.760639:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x60:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.760651:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x60:0x0] 00000008:00010000:3.0:1788876623.760672:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.760675:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9dbb0, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.760693:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.760707:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.760719:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.760723:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.760731:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.760741:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.760754:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x60:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.760758:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.760766:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.760769:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dbb0 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.760773:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.760777:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dbb0 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.760786:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x60:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.760817:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.760826:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.760834:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.760843:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.760850:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.760854:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.760857:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486d4 00000080:00010000:3.0:1788876623.760861:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486d4 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.760865:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.760873:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.760882:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.760888:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-54, flags=0 00000080:00200000:3.0:1788876623.760892:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.760899:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.760904:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.760912:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.760920:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.760924:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.760926:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486d4 00000080:00010000:3.0:1788876623.760930:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486d4 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.760933:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.760940:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.760948:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.760954:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-54, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.760977:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.761016:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b800/0x7d14dcdc37047da4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa894 expref: -99 pid: 33203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.761029:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x60:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.761059:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b800/0x7d14dcdc37047da4 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa894 expref: -99 pid: 33203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.761068:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.761075:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.761077:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897b800/0x7d14dcdc37047da4 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x60:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa894 expref: -99 pid: 33203 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.761096:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486d4 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313add8b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.761112:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.761120:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188962a00 x1875771511194752/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.761141:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962a00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511194752:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.762687:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.778750:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962a00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511194752:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.778811:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x60:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.778822:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.778847:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x60:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.778898:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x60:0x0](ffffa061982300c8) 00000008:00010000:3.0:1788876623.778932:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.779015:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.779035:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.779093:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.779102:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.779112:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.779132:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511194880:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.779209:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.787833:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511194880:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.787859:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.787871:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.787883:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.787894:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.787907:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.787915:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.787923:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.787933:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188961500 x1875771511194880/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.787952:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.787965:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.787971:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.788009:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.788016:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486db 00000080:00010000:3.0:1788876623.788022:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.788027:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486db from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.788030:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.788040:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.788064:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.788075:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-55, flags=8 00000080:00200008:3.0:1788876623.788086:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-55:[0x200001b71:0x61:0x0](ffffa06198231988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.788091:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x61:0x0](ffffa06198231988),name=f5.replay-single-55 00000002:00010000:3.0:1788876623.788111:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x61:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.788122:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492c00/0x7d14dcdc37047dab lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa8b7 expref: -99 pid: 33204 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.788133:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492c00/0x7d14dcdc37047dab lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa8b7 expref: -99 pid: 33204 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.788140:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.788146:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x61:0x0](ffffa06198231988) 00000080:00010000:3.0:1788876623.788149:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x61:0x0](ffffa06198231988) for lock 0x7d14dcdc37047dab 00000080:00010000:3.0:1788876623.788153:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047dab from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.788155:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492c00/0x7d14dcdc37047dab lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa8b7 expref: -99 pid: 33204 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.788163:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492c00/0x7d14dcdc37047dab lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa8b7 expref: -99 pid: 33204 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.788182:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x61:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.788192:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x61:0x0] 00000008:00010000:3.0:1788876623.788211:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.788214:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9dbb0, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.788231:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007822ec6e/0x7d14dcdc37046e46 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a60c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.788242:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007822ec6e/0x7d14dcdc37046e46 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a60c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.788254:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.788258:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007822ec6e/0x7d14dcdc37046e46 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a60c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.788266:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007822ec6e/0x7d14dcdc37046e46 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a60c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.788276:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.788286:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x61:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.788290:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.788293:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.788296:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dbb0 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.788301:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.788303:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dbb0 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.788312:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x61:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.788343:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.788350:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.788357:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.788365:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.788372:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.788375:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.788378:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486db 00000080:00010000:3.0:1788876623.788422:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486db from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.788427:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.788434:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.788444:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.788451:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-55, flags=0 00000080:00200000:3.0:1788876623.788455:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.788462:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.788469:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.788477:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.788484:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.788487:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.788490:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486db 00000080:00010000:3.0:1788876623.788493:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486db from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.788495:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.788502:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.788509:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.788514:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-55, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.788535:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.788642:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492c00/0x7d14dcdc37047dab lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa8b7 expref: -99 pid: 33204 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.788653:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x61:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.788689:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492c00/0x7d14dcdc37047dab lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa8b7 expref: -99 pid: 33204 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.788697:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.788703:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.788705:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492c00/0x7d14dcdc37047dab lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x61:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa8b7 expref: -99 pid: 33204 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.788722:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486db lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313addb5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.788739:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.788747:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188961500 x1875771511195136/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.788767:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511195136:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.788831:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.793981:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511195136:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.794059:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x61:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.794072:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.794101:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x61:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.794146:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x61:0x0](ffffa06198231988) 00000008:00010000:3.0:1788876623.794183:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007822ec6e/0x7d14dcdc37046e46 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a60c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.794262:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.794282:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.794340:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.794354:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.794364:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.794414:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511195264:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.794473:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.797837:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511195264:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.797867:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.797880:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.797893:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.797904:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.797917:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.797926:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.797940:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.797952:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188963800 x1875771511195264/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.797968:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.797982:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.797989:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.798026:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.798032:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486e2 00000080:00010000:3.0:1788876623.798038:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.798041:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486e2 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.798045:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.798054:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.798080:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.798091:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-56, flags=8 00000080:00200008:3.0:1788876623.798102:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-56:[0x200001b71:0x62:0x0](ffffa06198239988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.798108:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x62:0x0](ffffa06198239988),name=f5.replay-single-56 00000002:00010000:3.0:1788876623.798128:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x62:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.798139:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825400/0x7d14dcdc37047db2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa8da expref: -99 pid: 33205 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.798151:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825400/0x7d14dcdc37047db2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa8da expref: -99 pid: 33205 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.798158:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.798162:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x62:0x0](ffffa06198239988) 00000080:00010000:3.0:1788876623.798165:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x62:0x0](ffffa06198239988) for lock 0x7d14dcdc37047db2 00000080:00010000:3.0:1788876623.798170:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047db2 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.798173:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825400/0x7d14dcdc37047db2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa8da expref: -99 pid: 33205 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.798195:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188825400/0x7d14dcdc37047db2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x62:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa8da expref: -99 pid: 33205 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.798217:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x62:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.798227:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x62:0x0] 00000008:00010000:3.0:1788876623.798247:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.798250:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9dbb0, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.798266:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.798278:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.798290:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.798294:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.798308:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.798319:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.798329:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x62:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.798333:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.798336:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.798339:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dbb0 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.798343:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.798345:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dbb0 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.798355:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x62:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.798420:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.798429:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.798437:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.798446:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.798453:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.798458:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.798461:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486e2 00000080:00010000:3.0:1788876623.798465:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486e2 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.798469:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.798482:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.798491:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.798498:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-56, flags=0 00000080:00200000:3.0:1788876623.798502:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.798508:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.798513:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.798521:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.798528:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.798532:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.798534:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486e2 00000080:00010000:3.0:1788876623.798538:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486e2 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.799526:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.799602:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc370486e2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adddf expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.799618:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.863749:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.863758:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486f7 00000080:00010000:3.0:1788876623.863764:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.863768:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486f7 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.863772:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.863782:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.863809:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.863819:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-59, flags=8 00000080:00200008:3.0:1788876623.863830:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-59:[0x200001b71:0x65:0x0](ffffa06198232a08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.863836:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x65:0x0](ffffa06198232a08),name=f5.replay-single-59 00000002:00010000:3.0:1788876623.863856:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x65:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.863867:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934000/0x7d14dcdc37047dc7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa943 expref: -99 pid: 33208 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.863879:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934000/0x7d14dcdc37047dc7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa943 expref: -99 pid: 33208 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.863887:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.863891:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x65:0x0](ffffa06198232a08) 00000080:00010000:3.0:1788876623.863894:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x65:0x0](ffffa06198232a08) for lock 0x7d14dcdc37047dc7 00000080:00010000:3.0:1788876623.863898:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047dc7 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.863902:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934000/0x7d14dcdc37047dc7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa943 expref: -99 pid: 33208 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.863909:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934000/0x7d14dcdc37047dc7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa943 expref: -99 pid: 33208 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.863933:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x65:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:3.0:1788876623.863954:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x65:0x0] 00000008:00010000:3.0:1788876623.863973:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.863977:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9dbb0, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.863992:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.864024:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.864037:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.864040:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.864048:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.864058:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.864069:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x65:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.864073:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.864076:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.864080:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dbb0 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.864084:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.864086:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dbb0 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.864097:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x65:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.864130:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.864138:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.864145:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.864153:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.864160:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.864163:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.864165:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486f7 00000080:00010000:3.0:1788876623.864169:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486f7 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.864171:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.864178:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.864186:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.864192:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-59, flags=0 00000080:00200000:3.0:1788876623.864195:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.864200:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.864205:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.864211:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.864217:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.864220:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.864223:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486f7 00000080:00010000:3.0:1788876623.864226:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486f7 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.864228:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.864234:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.864240:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.864245:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-59, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.864265:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.864294:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934000/0x7d14dcdc37047dc7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa943 expref: -99 pid: 33208 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.864304:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x65:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.864337:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934000/0x7d14dcdc37047dc7 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa943 expref: -99 pid: 33208 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.864344:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.864350:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.864353:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188934000/0x7d14dcdc37047dc7 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x65:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa943 expref: -99 pid: 33208 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.864371:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc370486f7 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ade5d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.864443:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.864453:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188963800 x1875771511196672/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.864477:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511196672:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.864555:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.877034:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511196672:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.877107:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x65:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.877123:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.877155:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x65:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.877208:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x65:0x0](ffffa06198232a08) 00000008:00010000:3.0:1788876623.877246:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.877326:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.877346:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.877474:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.877484:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.877494:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.877516:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511196800:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.877579:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.896736:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511196800:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.896766:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896780:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896795:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896807:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896820:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896829:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896848:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.896862:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188962d80 x1875771511196800/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.896876:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896894:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.896902:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.896927:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.896933:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486fe 00000080:00010000:3.0:1788876623.896939:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.896942:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486fe from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.896946:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.896956:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.896982:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.896993:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-6, flags=8 00000080:00200008:3.0:1788876623.897015:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-6:[0x200001b71:0x30:0x0](ffffa0619817c2c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.897021:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x30:0x0](ffffa0619817c2c8),name=f5.replay-single-6 00000002:00010000:3.0:1788876623.897041:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x30:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.897051:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935c00/0x7d14dcdc37047c1c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa11d expref: -99 pid: 33151 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.897063:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935c00/0x7d14dcdc37047c1c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa11d expref: -99 pid: 33151 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788876623.897071:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.897075:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x30:0x0](ffffa0619817c2c8) 00000080:00010000:3.0:1788876623.897078:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x30:0x0](ffffa0619817c2c8) for lock 0x7d14dcdc37047c1c 00000080:00010000:3.0:1788876623.897083:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047c1c from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.897086:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935c00/0x7d14dcdc37047c1c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa11d expref: -99 pid: 33151 timeout: 0 lvb_type: 3 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.897095:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935c00/0x7d14dcdc37047c1c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa11d expref: -99 pid: 33151 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00000080:00200000:3.0:1788876623.897117:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x30:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.897127:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x30:0x0] 00000008:00010000:3.0:1788876623.897148:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.897151:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9de30, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.897169:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1788876623.897181:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00000020:00200000:3.0:1788876623.897194:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.897198:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1788876623.897208:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000020:00200000:3.0:1788876623.897218:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.897230:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x30:0x0] updating i_size 6 i_blocks 8 00000020:00010000:3.0:1788876623.897234:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.897238:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.897241:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9de30 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.897245:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.897248:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9de30 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.897258:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x30:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.897288:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.897296:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.897303:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.897312:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.897319:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.897323:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.897326:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486fe 00000080:00010000:3.0:1788876623.897330:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486fe from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.897334:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.897342:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.897351:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.897357:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-6, flags=0 00000080:00200000:3.0:1788876623.897361:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.897366:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.897372:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.897434:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.897441:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.897445:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.897447:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370486fe 00000080:00010000:3.0:1788876623.897451:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370486fe from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.897454:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.897461:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.897471:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.897477:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-6, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.897500:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.897528:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935c00/0x7d14dcdc37047c1c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa11d expref: -99 pid: 33151 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00000080:00200000:3.0:1788876623.897538:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x30:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.897573:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935c00/0x7d14dcdc37047c1c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa11d expref: -99 pid: 33151 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1788876623.897581:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.897590:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.897593:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188935c00/0x7d14dcdc37047c1c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x30:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa11d expref: -99 pid: 33151 timeout: 0 lvb_type: 3 lru_score: 5 lru_type: 1 00010000:00010000:3.0:1788876623.897611:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370486fe lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ade87 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.897629:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.897638:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188962d80 x1875771511197056/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876623.897659:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197056:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876623.897731:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1180 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.905532:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197056:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.905602:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x30:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.905617:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.905648:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x30:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.905708:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x30:0x0](ffffa0619817c2c8) 00000008:00010000:1.0:1788876623.905768:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 6 blocks 8 [cma]time: 1788876558 1788876558 0 by oinfo size 6 blocks 8 [cma]time 1788876558 1788876558 1788876613 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1788876623.905860:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.905880:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.905939:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.905948:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.905957:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.905976:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197184:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.906192:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.910169:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197184:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.910202:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910217:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910231:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910244:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910282:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910291:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910300:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.910311:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188846680 x1875771511197184/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.910326:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910341:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.910348:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.910436:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.910444:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048705 00000080:00010000:1.0:1788876623.910449:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.910453:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048705 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.910456:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.910465:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.910501:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.910512:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-60, flags=8 00000080:00200008:1.0:1788876623.910523:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-60:[0x200001b71:0x66:0x0](ffffa0619823e3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.910529:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x66:0x0](ffffa0619823e3c8),name=f5.replay-single-60 00000002:00010000:1.0:1788876623.910551:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x66:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.910561:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492600/0x7d14dcdc37047dce lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x66:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa966 expref: -99 pid: 33209 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.910573:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492600/0x7d14dcdc37047dce lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x66:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa966 expref: -99 pid: 33209 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.910581:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.910585:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x66:0x0](ffffa0619823e3c8) 00000080:00010000:1.0:1788876623.910588:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x66:0x0](ffffa0619823e3c8) for lock 0x7d14dcdc37047dce 00000080:00010000:1.0:1788876623.910592:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047dce from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.910595:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492600/0x7d14dcdc37047dce lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x66:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa966 expref: -99 pid: 33209 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.910602:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492600/0x7d14dcdc37047dce lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x66:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa966 expref: -99 pid: 33209 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.910623:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x66:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.910633:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x66:0x0] 00000008:00010000:1.0:1788876623.910653:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.910656:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.910673:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.910693:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.910707:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.910711:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.910718:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.910729:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.910742:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x66:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.910746:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.910750:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.910753:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.910757:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.910761:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.910772:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x66:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.910802:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.910811:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.910818:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.910827:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.910844:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.910848:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.910851:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048705 00000080:00010000:1.0:1788876623.910856:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048705 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.910859:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.910867:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.910877:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.910884:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-60, flags=0 00000080:00200000:1.0:1788876623.910888:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.910893:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.910899:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.910907:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.910914:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.910918:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.910920:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048705 00000080:00010000:1.0:1788876623.910924:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048705 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.910927:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.910944:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.910953:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.910959:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-60, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.910981:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.911024:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492600/0x7d14dcdc37047dce lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x66:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa966 expref: -99 pid: 33209 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.911036:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x66:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.911067:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492600/0x7d14dcdc37047dce lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x66:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa966 expref: -99 pid: 33209 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.911076:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.911083:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.911086:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492600/0x7d14dcdc37047dce lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x66:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa966 expref: -99 pid: 33209 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.911105:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc37048705 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adeb1 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.911122:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.911131:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188846680 x1875771511197440/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.911157:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197440:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.911233:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.926304:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846680 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197440:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.926369:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x66:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.926434:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.926461:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x66:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.926509:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x66:0x0](ffffa0619823e3c8) 00000008:00010000:1.0:1788876623.926543:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876615 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.926620:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.926638:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.926698:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.926706:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.926728:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.926748:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197568:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.926804:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.938036:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197568:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.938076:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938107:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938121:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938133:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938146:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938155:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938163:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.938176:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188844a80 x1875771511197568/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.938191:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938205:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.938212:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876623.938245:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876623.938257:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704870c 00000080:00010000:1.0:1788876623.938263:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.938266:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704870c from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876623.938270:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.938279:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.938306:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.938319:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-61, flags=8 00000080:00200008:1.0:1788876623.938333:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-61:[0x200001b71:0x67:0x0](ffffa06198235b88), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876623.938340:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x67:0x0](ffffa06198235b88),name=f5.replay-single-61 00000002:00010000:1.0:1788876623.938365:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x67:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.938377:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3000/0x7d14dcdc37047dd5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa989 expref: -99 pid: 33210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.938425:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3000/0x7d14dcdc37047dd5 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa989 expref: -99 pid: 33210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.938433:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.938438:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x67:0x0](ffffa06198235b88) 00000080:00010000:1.0:1788876623.938441:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x67:0x0](ffffa06198235b88) for lock 0x7d14dcdc37047dd5 00000080:00010000:1.0:1788876623.938445:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047dd5 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876623.938448:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3000/0x7d14dcdc37047dd5 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa989 expref: -99 pid: 33210 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.938455:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3000/0x7d14dcdc37047dd5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa989 expref: -99 pid: 33210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.938481:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x67:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876623.938491:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x67:0x0] 00000008:00010000:1.0:1788876623.938511:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876623.938515:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2430, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876623.938532:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.938543:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876623.938554:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876623.938558:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876623.938565:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876623.938574:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876623.938585:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x67:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876623.938589:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1788876623.938593:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.938596:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2430 (ffffa06189032098/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876623.938599:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d720 (ffffa06189032ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876623.938601:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2430 (ffffa06189032ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876623.938612:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x67:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.938643:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.938658:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.938665:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.938673:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876623.938679:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.938683:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.938686:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704870c 00000080:00010000:1.0:1788876623.938690:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704870c from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876623.938693:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876623.938840:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.938850:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876623.938858:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-61, flags=0 00000080:00200000:1.0:1788876623.938862:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.938870:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.938876:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.938884:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876623.938890:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876623.938894:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876623.938901:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704870c 00000080:00010000:1.0:1788876623.938905:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704870c from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876623.938908:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876623.938915:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.938923:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876623.938929:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-61, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876623.938950:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.938979:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3000/0x7d14dcdc37047dd5 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa989 expref: -99 pid: 33210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876623.938990:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x67:0x0]: losing layout lock 00010000:00010000:1.0:1788876623.939037:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3000/0x7d14dcdc37047dd5 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa989 expref: -99 pid: 33210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.939046:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.939262:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876623.939266:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3000/0x7d14dcdc37047dd5 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x67:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa989 expref: -99 pid: 33210 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876623.939290:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc3704870c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adedb expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876623.939321:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876623.939331:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188844a80 x1875771511197824/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876623.939353:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197824:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876623.939531:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.949274:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197824:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876623.949335:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x67:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876623.949348:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876623.949402:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x67:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876623.949452:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x67:0x0](ffffa06198235b88) 00000008:00010000:1.0:1788876623.949485:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876623.949559:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876623.949580:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876623.949635:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.949644:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.949653:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876623.949673:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197952:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876623.949754:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:1.0:1788876623.960952:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511197952:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876623.960982:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.960995:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.961687:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.961700:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.961716:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.961725:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.961732:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876623.961746:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188844380 x1875771511197952/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876623.961760:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc37048713 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf05 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876623.961795:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876623.961801:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000100:00100000:3.0:1788876623.970788:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ecca80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511198208:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876623.970861:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x68:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876623.970875:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876623.970900:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x68:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.970947:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x68:0x0](ffffa061982321c8) 00000008:00010000:3.0:1788876623.970981:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 0 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 0 [cma]time 1788876560 1788876560 1788876616 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008d0e0c0b/0x7d14dcdc37046ed9 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a624e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876623.971142:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.971163:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.971227:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.971241:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.971251:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876623.971287:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961880 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511198336:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876623.971364:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876623.979067:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961880 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511198336:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876623.979109:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979122:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979136:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979147:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979162:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979171:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979186:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876623.979199:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188961880 x1875771511198336/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876639 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876623.979213:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979229:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876623.979236:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876623.979264:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876623.979270:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704871a 00000080:00010000:3.0:1788876623.979276:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.979280:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704871a from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876623.979284:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876623.979294:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.979320:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.979331:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-63, flags=8 00000080:00200008:3.0:1788876623.979343:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-63:[0x200001b71:0x69:0x0](ffffa06198235348), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876623.979349:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x69:0x0](ffffa06198235348),name=f5.replay-single-63 00000002:00010000:3.0:1788876623.979366:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x69:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.979376:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492200/0x7d14dcdc37047de3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x69:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa9cf expref: -99 pid: 33212 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.979420:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492200/0x7d14dcdc37047de3 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x69:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa9cf expref: -99 pid: 33212 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.979428:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.979433:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x69:0x0](ffffa06198235348) 00000080:00010000:3.0:1788876623.979437:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x69:0x0](ffffa06198235348) for lock 0x7d14dcdc37047de3 00000080:00010000:3.0:1788876623.979441:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047de3 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876623.979444:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492200/0x7d14dcdc37047de3 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x69:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aa9cf expref: -99 pid: 33212 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.979452:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492200/0x7d14dcdc37047de3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x69:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aa9cf expref: -99 pid: 33212 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.979473:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x69:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876623.979482:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x69:0x0] 00000008:00010000:3.0:1788876623.979502:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876623.979506:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9de30, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876623.979523:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.979534:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876623.979545:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876623.979549:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876623.979561:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876623.979570:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876623.979581:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x69:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876623.979585:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876623.979588:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.979591:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9de30 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876623.979595:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876623.979598:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9de30 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876623.979607:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x69:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876623.979637:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.979645:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.979653:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.979662:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876623.979681:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.979686:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.979689:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704871a 00000080:00010000:3.0:1788876623.979693:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704871a from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876623.979696:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876623.979852:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.979864:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876623.979872:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-63, flags=0 00000080:00200000:3.0:1788876623.979877:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876623.979884:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876623.979892:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.979901:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876623.979908:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876623.979912:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876623.979915:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704871a 00000080:00010000:3.0:1788876623.979919:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704871a from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876623.979922:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876623.979930:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.979937:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876623.979943:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-63, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876623.979965:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.979993:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492200/0x7d14dcdc37047de3 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x69:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aa9cf expref: -99 pid: 33212 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876623.980015:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x69:0x0]: losing layout lock 00010000:00010000:3.0:1788876623.980053:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492200/0x7d14dcdc37047de3 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x69:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa9cf expref: -99 pid: 33212 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.980061:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.980067:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876623.980069:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06181492200/0x7d14dcdc37047de3 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x69:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aa9cf expref: -99 pid: 33212 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876623.980361:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704871a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313adf2f expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876623.980422:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876623.980431:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188961880 x1875771511198592/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000080:00200000:3.0:1788876624.080618:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876624.080631:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704873d 00000080:00010000:3.0:1788876624.080637:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.080640:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704873d from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876624.080645:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.080667:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.080696:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.080716:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-68, flags=8 00000080:00200008:3.0:1788876624.080728:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-68:[0x200001b71:0x6e:0x0](ffffa0619823aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876624.080734:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x6e:0x0](ffffa0619823aa08),name=f5.replay-single-68 00000002:00010000:3.0:1788876624.080756:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x6e:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.080769:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897be00/0x7d14dcdc37047e06 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaa7e expref: -99 pid: 33217 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.080780:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897be00/0x7d14dcdc37047e06 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaa7e expref: -99 pid: 33217 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.080788:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.080793:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x6e:0x0](ffffa0619823aa08) 00000080:00010000:3.0:1788876624.080796:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x6e:0x0](ffffa0619823aa08) for lock 0x7d14dcdc37047e06 00000080:00010000:3.0:1788876624.080800:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047e06 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876624.080803:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897be00/0x7d14dcdc37047e06 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x6e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aaa7e expref: -99 pid: 33217 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.080811:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897be00/0x7d14dcdc37047e06 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaa7e expref: -99 pid: 33217 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.080831:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x6e:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:3.0:1788876624.080931:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x6e:0x0] 00000008:00010000:3.0:1788876624.080952:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876624.080956:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9dbb0, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876624.080974:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.080986:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876624.081008:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876624.081013:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.081021:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876624.081032:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876624.081044:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x6e:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876624.081048:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_release():245 00000020:00010000:3.0:1788876624.081051:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.081054:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dbb0 (ffffa06189032998/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.081057:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061887a5970 (ffffa06188721d88/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876624.081060:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dbb0 (ffffa06188721d88/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876624.081071:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x6e:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.081103:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.081112:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.081119:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.081133:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.081139:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.081143:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.081146:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704873d 00000080:00010000:3.0:1788876624.081151:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704873d from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876624.081153:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.081161:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.081170:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.081176:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-68, flags=0 00000080:00200000:3.0:1788876624.081180:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.081187:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.081193:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.081201:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.081208:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.081668:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.081671:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704873d 00000080:00010000:3.0:1788876624.081676:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704873d from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876624.081681:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.081690:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.081700:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876624.081706:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-68, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876624.082360:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.082995:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897be00/0x7d14dcdc37047e06 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aaa7e expref: -99 pid: 33217 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.083021:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x6e:0x0]: losing layout lock 00010000:00010000:3.0:1788876624.083066:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897be00/0x7d14dcdc37047e06 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x6e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aaa7e expref: -99 pid: 33217 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.083075:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.083081:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876624.083085:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897be00/0x7d14dcdc37047e06 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x6e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aaa7e expref: -99 pid: 33217 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.083106:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc3704873d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae001 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876624.083125:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876624.083138:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188963b80 x1875771511200512/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876624.083160:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963b80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511200512:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876624.083299:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.104375:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963b80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511200512:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876624.104584:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x6e:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876624.104598:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876624.104627:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x6e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.104684:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x6e:0x0](ffffa0619823aa08) 00000008:00010000:3.0:1788876624.104732:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876616 ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.104817:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.104898:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.104957:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.104965:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.104974:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876624.104992:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511200640:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876624.105067:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.124467:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511200640:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876624.124499:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124511:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124525:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124536:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124550:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124559:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124566:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876624.124578:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188963480 x1875771511200640/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876624.124593:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124607:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876624.124613:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876624.124644:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876624.124650:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048744 00000080:00010000:3.0:1788876624.124655:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.124658:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048744 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876624.124661:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.124670:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048744 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae02b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.124695:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.124705:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-69, flags=8 00000080:00200008:3.0:1788876624.124717:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-69:[0x200001b71:0x6f:0x0](ffffa061982400c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876624.124722:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x6f:0x0](ffffa061982400c8),name=f5.replay-single-69 00000002:00010000:3.0:1788876624.124742:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x6f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.124754:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897ac00/0x7d14dcdc37047e0d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaaa1 expref: -99 pid: 33218 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.124765:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897ac00/0x7d14dcdc37047e0d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x6f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaaa1 expref: -99 pid: 33218 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.124773:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.124778:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x6f:0x0](ffffa061982400c8) 00000080:00010000:3.0:1788876624.124780:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x6f:0x0](ffffa061982400c8) for lock 0x7d14dcdc37047e0d 00000080:00010000:3.0:1788876624.124784:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047e0d from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876624.124787:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897ac00/0x7d14dcdc37047e0d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x6f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aaaa1 expref: -99 pid: 33218 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.124799:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897ac00/0x7d14dcdc37047e0d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x6f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaaa1 expref: -99 pid: 33218 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.124820:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x6f:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876624.124829:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x6f:0x0] 00000008:00010000:3.0:1788876624.124848:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876624.124851:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9de30, osc lock ffffa061885bfa20, flags 0x400001002 00010000:00010000:3.0:1788876624.124868:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.124879:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876624.124891:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876624.124895:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.124903:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876624.125525:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00010000:00010000:3.0:1788876624.188561:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3c00/0x7d14dcdc37047e1b lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaae7 expref: -99 pid: 33220 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.188571:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.188578:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x71:0x0](ffffa06198241988) 00000080:00010000:3.0:1788876624.188582:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x71:0x0](ffffa06198241988) for lock 0x7d14dcdc37047e1b 00000080:00010000:3.0:1788876624.188586:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047e1b from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876624.188590:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3c00/0x7d14dcdc37047e1b lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aaae7 expref: -99 pid: 33220 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.188597:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3c00/0x7d14dcdc37047e1b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaae7 expref: -99 pid: 33220 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.188618:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x71:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876624.188628:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x71:0x0] 00000008:00010000:3.0:1788876624.188647:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876624.188651:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9dc30, osc lock ffffa061885be4c0, flags 0x400001002 00010000:00010000:3.0:1788876624.188728:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.188742:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876624.188755:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876624.188760:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.188769:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876624.188779:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876624.188790:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x71:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876624.188794:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615ee40 (ffffa06189032878/0) at cl_lock_release():245 00000020:00010000:3.0:1788876624.188798:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615ee40 (ffffa06189032878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.188802:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dc30 (ffffa061890322d8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.188806:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615ee40 (ffffa06189032878/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876624.188809:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dc30 (ffffa06189032878/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876624.188819:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x71:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.188849:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.188858:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.188866:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.188875:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.188882:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.188888:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.188891:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048759 00000080:00010000:3.0:1788876624.188895:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048759 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876624.188899:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.188907:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.188916:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.188924:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-71, flags=0 00000080:00200000:3.0:1788876624.188928:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.188934:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.188941:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.188950:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.188957:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.188961:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.188964:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048759 00000080:00010000:3.0:1788876624.188967:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048759 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876624.188970:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.188979:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.188987:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876624.188993:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-71, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876624.189025:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.189052:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3c00/0x7d14dcdc37047e1b lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aaae7 expref: -99 pid: 33220 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.189063:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x71:0x0]: losing layout lock 00010000:00010000:3.0:1788876624.189161:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3c00/0x7d14dcdc37047e1b lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aaae7 expref: -99 pid: 33220 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.189171:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.189185:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876624.189188:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3c00/0x7d14dcdc37047e1b lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x71:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aaae7 expref: -99 pid: 33220 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.189209:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc37048759 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae0a9 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876624.189227:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876624.189235:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188963480 x1875771511202048/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876624.189256:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511202048:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876624.189374:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.195918:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511202048:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876624.196053:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x71:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876624.196067:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876624.196096:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x71:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.196146:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x71:0x0](ffffa06198241988) 00000008:00010000:3.0:1788876624.196181:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876616 ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.196259:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.196278:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.196336:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.196351:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.196362:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876624.196456:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511202176:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876624.196565:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.204475:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511202176:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876624.204506:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204520:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204535:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204548:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204563:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204572:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204588:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876624.204601:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188962300 x1875771511202176/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876624.204615:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204631:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876624.204637:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876624.204665:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876624.204672:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048760 00000080:00010000:3.0:1788876624.204677:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.204681:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048760 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876624.204684:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.204695:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.204737:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.204749:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-72, flags=8 00000080:00200008:3.0:1788876624.204760:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-72:[0x200001b71:0x72:0x0](ffffa061982463c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876624.204765:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x72:0x0](ffffa061982463c8),name=f5.replay-single-72 00000002:00010000:3.0:1788876624.204785:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x72:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.204795:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897a000/0x7d14dcdc37047e22 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aab0a expref: -99 pid: 33221 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.204807:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897a000/0x7d14dcdc37047e22 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aab0a expref: -99 pid: 33221 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.204813:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.204817:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x72:0x0](ffffa061982463c8) 00000080:00010000:3.0:1788876624.204821:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x72:0x0](ffffa061982463c8) for lock 0x7d14dcdc37047e22 00000080:00010000:3.0:1788876624.204825:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047e22 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876624.204828:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897a000/0x7d14dcdc37047e22 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aab0a expref: -99 pid: 33221 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.205132:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897a000/0x7d14dcdc37047e22 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aab0a expref: -99 pid: 33221 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.205156:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x72:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876624.205166:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x72:0x0] 00000008:00010000:3.0:1788876624.205189:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876624.205193:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9deb0, osc lock ffffa061885be4c0, flags 0x400001002 00010000:00010000:3.0:1788876624.205211:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.205223:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876624.205235:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876624.205239:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.205251:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876624.205261:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876624.205271:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x72:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876624.205276:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615ee40 (ffffa06189032878/0) at cl_lock_release():245 00000020:00010000:3.0:1788876624.205279:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615ee40 (ffffa06189032878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.205282:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9deb0 (ffffa061890322d8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.205285:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615ee40 (ffffa06189032878/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876624.205289:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9deb0 (ffffa06189032878/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876624.205297:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x72:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.205327:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.205335:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.205342:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.205350:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.205356:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.205360:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.205363:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048760 00000080:00010000:3.0:1788876624.205367:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048760 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876624.205370:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.206216:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.206233:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.206243:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-72, flags=0 00000080:00200000:3.0:1788876624.206521:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.206535:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.206545:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.206557:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.206565:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.206570:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.206573:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048760 00000080:00010000:3.0:1788876624.206579:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048760 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876624.206582:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.206590:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.206600:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876624.206607:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-72, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876624.206636:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.206667:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897a000/0x7d14dcdc37047e22 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aab0a expref: -99 pid: 33221 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.206679:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x72:0x0]: losing layout lock 00010000:00010000:3.0:1788876624.206737:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897a000/0x7d14dcdc37047e22 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aab0a expref: -99 pid: 33221 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.206746:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.206753:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876624.206756:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897a000/0x7d14dcdc37047e22 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x72:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aab0a expref: -99 pid: 33221 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.206778:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048760 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae0d3 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876624.206797:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876624.206808:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188962300 x1875771511202432/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876624.206844:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188962300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511202432:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876624.206924:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.214185:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188962300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511202432:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876624.214246:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x72:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876624.214259:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876624.214288:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x72:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.214337:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x72:0x0](ffffa061982463c8) 00000008:00010000:3.0:1788876624.214372:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876616 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.251164:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.251176:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.251185:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.251194:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.251201:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.251205:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.251208:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048775 00000080:00010000:3.0:1788876624.251211:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048775 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876624.251215:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.251222:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.251231:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.251239:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-75, flags=0 00000080:00200000:3.0:1788876624.251243:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.251250:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.251257:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.251264:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.251270:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.251274:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.251276:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048775 00000080:00010000:3.0:1788876624.251279:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048775 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876624.251282:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.251289:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.251297:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876624.251303:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-75, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876624.251327:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.251353:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3600/0x7d14dcdc37047e37 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aab73 expref: -99 pid: 33224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.251365:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x75:0x0]: losing layout lock 00010000:00010000:3.0:1788876624.251439:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3600/0x7d14dcdc37047e37 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aab73 expref: -99 pid: 33224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.251449:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.251460:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876624.251463:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061888d3600/0x7d14dcdc37047e37 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x75:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aab73 expref: -99 pid: 33224 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.251485:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048775 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae151 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876624.251504:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876624.251512:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188963100 x1875771511203584/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876624.251534:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188963100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511203584:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876624.251613:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.257938:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188963100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511203584:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876624.258016:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x75:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788876624.258030:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876624.258060:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x75:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.258960:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x75:0x0](ffffa06198245b88) 00000008:00010000:3.0:1788876624.259014:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876616 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004efb374d/0x7d14dcdc37046fea lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6526 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.259124:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.259150:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.259218:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.259231:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.259240:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876624.259263:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511203712:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876624.259335:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.266907:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511203712:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876624.266938:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.266951:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.266965:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.266977:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.266992:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.267011:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.267026:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876624.267039:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188961f80 x1875771511203712/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876624.267053:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.267068:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876624.267239:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876624.267275:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876624.267282:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704877c 00000080:00010000:3.0:1788876624.267289:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.267292:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704877c from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876624.267296:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.267307:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.267336:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.267348:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-76, flags=8 00000080:00200008:3.0:1788876624.267360:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-76:[0x200001b71:0x76:0x0](ffffa0619823a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876624.267368:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x76:0x0](ffffa0619823a1c8),name=f5.replay-single-76 00000002:00010000:3.0:1788876624.267474:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x76:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.267486:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897bc00/0x7d14dcdc37047e3e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aab96 expref: -99 pid: 33225 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.267499:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897bc00/0x7d14dcdc37047e3e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aab96 expref: -99 pid: 33225 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.267506:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.267511:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x76:0x0](ffffa0619823a1c8) 00000080:00010000:3.0:1788876624.267514:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x76:0x0](ffffa0619823a1c8) for lock 0x7d14dcdc37047e3e 00000080:00010000:3.0:1788876624.267518:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047e3e from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876624.267522:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897bc00/0x7d14dcdc37047e3e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aab96 expref: -99 pid: 33225 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.267530:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897bc00/0x7d14dcdc37047e3e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aab96 expref: -99 pid: 33225 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.267554:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x76:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876624.267567:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x76:0x0] 00000008:00010000:3.0:1788876624.267593:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876624.267598:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9deb0, osc lock ffffa061885be4c0, flags 0x400001002 00010000:00010000:3.0:1788876624.267621:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.267633:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876624.267646:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876624.267650:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.267665:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876624.267677:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876624.267689:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x76:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876624.267694:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788876624.267697:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.267701:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9deb0 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.267720:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876624.267724:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9deb0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876624.267735:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x76:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.267769:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.267778:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.267787:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.267795:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.267801:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.267805:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.267808:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704877c 00000080:00010000:3.0:1788876624.267811:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704877c from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876624.267814:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.267825:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.267832:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.269601:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-76, flags=0 00000080:00200000:3.0:1788876624.269609:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.269627:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.269642:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.269656:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.269663:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.269668:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.269680:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704877c 00000080:00010000:3.0:1788876624.269687:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704877c from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876624.269690:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.269699:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.270016:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876624.270024:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-76, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876624.270056:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.270101:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897bc00/0x7d14dcdc37047e3e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aab96 expref: -99 pid: 33225 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.270112:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x76:0x0]: losing layout lock 00010000:00010000:3.0:1788876624.270152:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897bc00/0x7d14dcdc37047e3e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aab96 expref: -99 pid: 33225 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.270161:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.270167:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876624.270171:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618897bc00/0x7d14dcdc37047e3e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x76:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aab96 expref: -99 pid: 33225 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.270191:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc3704877c lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae17b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876624.270210:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876624.270223:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188961f80 x1875771511203968/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876624.270246:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188961f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511203968:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876624.270333:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.279555:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188961f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511203968:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876624.279628:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x76:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876624.279642:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876624.279681:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x76:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.279928:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x76:0x0](ffffa0619823a1c8) 00000008:00010000:3.0:1788876624.280028:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876616 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.280153:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.280173:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.280230:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.280239:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.280250:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876624.280270:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511204096:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876624.280346:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.283219:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511204096:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876624.283247:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.283261:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.283274:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.283288:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.283304:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.283314:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.283323:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876624.283336:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188871f80 x1875771511204096/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876624.283352:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc37048783 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae1a5 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.283366:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876624.283373:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000100:00100000:3.0:1788876624.350223:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618893f100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511205888:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876624.350327:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.353454:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618893f100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511205888:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876624.353516:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x7a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788876624.353528:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876624.353556:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x7a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.353608:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x7a:0x0](ffffa061982442c8) 00000008:00010000:3.0:1788876624.353643:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876616 ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009d1b7ad/0x7d14dcdc37047053 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1e9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a663e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.353740:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.353936:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.354011:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.354021:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.354032:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876624.354061:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511206016:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876624.354204:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.357424:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511206016:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876624.357450:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357464:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357478:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357490:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357509:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357518:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357526:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876624.357538:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188871c00 x1875771511206016/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876624.357552:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357565:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876624.357571:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876624.357598:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876624.357604:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370487a6 00000080:00010000:3.0:1788876624.357609:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.357612:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370487a6 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876624.357615:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.357625:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.357656:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.357666:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-81, flags=8 00000080:00200008:3.0:1788876624.357677:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-81:[0x200001b71:0x7b:0x0](ffffa06198244b08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876624.357683:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x7b:0x0](ffffa06198244b08),name=f5.replay-single-81 00000002:00010000:3.0:1788876624.357702:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x7b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.357724:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187b75600/0x7d14dcdc37047e61 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aac45 expref: -99 pid: 33230 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.357736:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187b75600/0x7d14dcdc37047e61 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aac45 expref: -99 pid: 33230 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.357743:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.357748:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x7b:0x0](ffffa06198244b08) 00000080:00010000:3.0:1788876624.357751:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x7b:0x0](ffffa06198244b08) for lock 0x7d14dcdc37047e61 00000080:00010000:3.0:1788876624.357754:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047e61 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876624.357758:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187b75600/0x7d14dcdc37047e61 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aac45 expref: -99 pid: 33230 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.357766:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06187b75600/0x7d14dcdc37047e61 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x7b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aac45 expref: -99 pid: 33230 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.357786:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x7b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788876624.357796:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x7b:0x0] 00000008:00010000:3.0:1788876624.357814:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876624.357818:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d930, osc lock ffffa061885be4c0, flags 0x400001002 00010000:00010000:3.0:1788876624.357845:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.357859:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876624.357871:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876624.357876:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.357883:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876624.357893:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876624.357904:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x7b:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876624.357908:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788876624.357910:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.357913:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d930 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.357918:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876624.357920:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d930 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876624.357931:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x7b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.357961:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.357968:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.357976:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.357983:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.358005:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.358010:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.358012:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370487a6 00000080:00010000:3.0:1788876624.358015:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370487a6 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876624.358019:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.358025:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.358033:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.358039:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-81, flags=0 00000080:00200000:3.0:1788876624.358044:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.358050:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.358056:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.358064:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.358070:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.358074:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.358192:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370487a6 00000080:00010000:3.0:1788876624.358197:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370487a6 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876624.358200:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.358213:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b000/0x7d14dcdc370487a6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae277 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.358223:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000008:00010000:3.0:1788876624.492830:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876617 ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009dc2e470/0x7d14dcdc370470e6 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a67c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.492934:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.492956:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.493030:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.493040:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.493051:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876624.493085:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511208704:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876624.493168:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.496867:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511208704:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876624.496899:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.496913:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.496932:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.496944:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.496957:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.496966:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.496973:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876624.496984:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188871c00 x1875771511208704/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876624.497010:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.497025:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876624.497032:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876624.497060:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876624.497066:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370487d7 00000080:00010000:3.0:1788876624.497072:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.497086:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370487d7 from it ffffb95ac20a7b58 00010000:00010000:3.0:1788876624.497090:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876624.497100:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.497125:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.497136:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-88, flags=8 00000080:00200008:3.0:1788876624.497149:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-88:[0x200001b71:0x82:0x0](ffffa06198249988), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876624.497155:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x82:0x0](ffffa06198249988),name=f5.replay-single-88 00000002:00010000:3.0:1788876624.497174:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x82:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.497184:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b000/0x7d14dcdc37047e92 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aad3a expref: -99 pid: 33237 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.497196:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b000/0x7d14dcdc37047e92 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aad3a expref: -99 pid: 33237 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.497204:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.497208:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x82:0x0](ffffa06198249988) 00000080:00010000:3.0:1788876624.497211:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x82:0x0](ffffa06198249988) for lock 0x7d14dcdc37047e92 00000080:00010000:3.0:1788876624.497215:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047e92 from it ffffb95ac20a7cb8 00010000:00010000:3.0:1788876624.497218:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b000/0x7d14dcdc37047e92 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aad3a expref: -99 pid: 33237 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.497226:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b000/0x7d14dcdc37047e92 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aad3a expref: -99 pid: 33237 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.497246:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x82:0x0] ignore/verify layout 0/1, layout version 65535 restore needed 0 00000001:00010000:3.0:1788876624.497257:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x82:0x0] 00000008:00010000:3.0:1788876624.497278:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788876624.497281:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9df30, osc lock ffffa061885be4c0, flags 0x400001002 00010000:00010000:3.0:1788876624.497299:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.497312:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:3.0:1788876624.497324:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788876624.497328:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876624.497337:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:3.0:1788876624.497348:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788876624.497360:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x82:0x0] updating i_size 7 i_blocks 8 00000020:00010000:3.0:1788876624.497364:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788876624.497368:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.497371:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9df30 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788876624.497375:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788876624.497378:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9df30 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788876624.497415:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x82:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.497447:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.497456:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.497464:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.497473:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876624.497479:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.497483:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.497485:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370487d7 00000080:00010000:3.0:1788876624.497489:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370487d7 from it ffffb95ac20a7c10 00010000:00010000:3.0:1788876624.497492:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876624.497499:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.497507:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876624.497514:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-88, flags=0 00000080:00200000:3.0:1788876624.497517:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876624.497524:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876624.497529:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.497538:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876624.497544:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876624.497548:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876624.497550:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370487d7 00000080:00010000:3.0:1788876624.497554:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370487d7 from it ffffb95ac20a7d68 00010000:00010000:3.0:1788876624.497557:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876624.497565:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.497572:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876624.497577:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-88, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876624.497598:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.497623:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b000/0x7d14dcdc37047e92 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aad3a expref: -99 pid: 33237 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876624.497635:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x82:0x0]: losing layout lock 00010000:00010000:3.0:1788876624.497667:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b000/0x7d14dcdc37047e92 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aad3a expref: -99 pid: 33237 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.497675:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.497681:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876624.497684:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618890b000/0x7d14dcdc37047e92 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x82:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aad3a expref: -99 pid: 33237 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876624.498840:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370487d7 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae39d expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876624.498877:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876624.498889:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188871c00 x1875771511208960/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876624.498914:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511208960:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876624.499045:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f1600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876624.508437:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511208960:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876624.508497:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x82:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:3.0:1788876624.508510:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788876624.508536:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x82:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876624.508582:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x82:0x0](ffffa06198249988) 00000008:00010000:3.0:1788876624.508616:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876617 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876624.508692:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00020000:00200000:1.0:1788876624.575422:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876624.575450:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x84:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.575498:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x84:0x0](ffffa06198249148) 00000008:00010000:1.0:1788876624.575533:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876560 1788876560 0 by oinfo size 7 blocks 8 [cma]time 1788876560 1788876560 1788876617 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000466c7b4/0x7d14dcdc37047125 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1ee:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a686e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876624.575612:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.575632:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.575713:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370487f3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.575723:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370487f3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.575733:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc370487f3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876624.575755:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511210240:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876624.575823:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:1.0:1788876624.667342:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea4700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511212416:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876624.667457:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x8a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788876624.667470:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876624.667496:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x8a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.667546:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8a:0x0](ffffa0619824a1c8) 00000008:00010000:1.0:1788876624.667589:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 7 blocks 8 [cma]time 1788876561 1788876561 1788876617 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e066b60f/0x7d14dcdc370471a3 lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1f1:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a69be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876624.667667:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.667688:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.668862:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.668873:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.668883:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876624.668904:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511212544:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876624.669037:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876624.681165:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511212544:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876624.681198:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681213:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681226:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681237:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681281:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681289:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681296:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876624.681308:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188844380 x1875771511212544/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876624.681322:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681348:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876624.681355:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876624.681403:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876624.681410:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704881d 00000080:00010000:1.0:1788876624.681416:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.681419:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704881d from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876624.681423:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.681432:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876624.681470:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876624.681482:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-97, flags=8 00000080:00200008:1.0:1788876624.681493:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-97:[0x200001b71:0x8b:0x0](ffffa0619824d348), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876624.681499:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x8b:0x0](ffffa0619824d348),name=f5.replay-single-97 00000002:00010000:1.0:1788876624.681519:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x8b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.681531:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ac000/0x7d14dcdc37047ed1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aae75 expref: -99 pid: 33246 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.681543:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ac000/0x7d14dcdc37047ed1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aae75 expref: -99 pid: 33246 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.681550:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.681555:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x8b:0x0](ffffa0619824d348) 00000080:00010000:1.0:1788876624.681558:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x8b:0x0](ffffa0619824d348) for lock 0x7d14dcdc37047ed1 00000080:00010000:1.0:1788876624.681562:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047ed1 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876624.681565:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ac000/0x7d14dcdc37047ed1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aae75 expref: -99 pid: 33246 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.681584:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ac000/0x7d14dcdc37047ed1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aae75 expref: -99 pid: 33246 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.681606:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x8b:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876624.681616:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x8b:0x0] 00000008:00010000:1.0:1788876624.681635:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876624.681638:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2230, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876624.681655:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876624.682043:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876624.682055:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876624.682059:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876624.682069:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876624.682081:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876624.682093:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x8b:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876624.682097:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:1.0:1788876624.682100:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876624.682104:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2230 (ffffa06189032ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876624.682108:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876624.682110:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2230 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876624.682123:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x8b:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.682154:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.682164:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.682172:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876624.682181:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876624.682200:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.682204:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.682207:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704881d 00000080:00010000:1.0:1788876624.682212:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704881d from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876624.682215:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876624.682223:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.682233:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876624.682240:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-97, flags=0 00000080:00200000:1.0:1788876624.682244:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.682250:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.682257:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.682264:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.682270:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.682274:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.682276:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704881d 00000080:00010000:1.0:1788876624.682280:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704881d from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876624.682283:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.682301:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.682311:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876624.682316:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-97, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876624.682339:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.682362:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ac000/0x7d14dcdc37047ed1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aae75 expref: -99 pid: 33246 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.682372:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x8b:0x0]: losing layout lock 00010000:00010000:1.0:1788876624.682446:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ac000/0x7d14dcdc37047ed1 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aae75 expref: -99 pid: 33246 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.682455:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.682461:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876624.682465:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061887ac000/0x7d14dcdc37047ed1 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x8b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aae75 expref: -99 pid: 33246 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.682486:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704881d lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae541 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876624.682505:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876624.682514:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188844380 x1875771511212800/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876624.682542:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511212800:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876624.683816:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876624.692118:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511212800:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876624.692184:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x8b:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876624.692199:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876624.692228:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x8b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.692276:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8b:0x0](ffffa0619824d348) 00000008:00010000:1.0:1788876624.692311:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 7 blocks 8 [cma]time 1788876561 1788876561 1788876617 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876624.692427:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.692451:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.692512:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.692522:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.692531:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876624.692554:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b6300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511212928:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876624.692642:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876624.702566:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061887b6300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511212928:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876624.702605:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702632:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702646:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702657:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702787:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702796:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702804:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876624.702818:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061887b6300 x1875771511212928/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876624.702831:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702845:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876624.702852:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876624.702877:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876624.702893:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048824 00000080:00010000:1.0:1788876624.702898:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.702901:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048824 from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876624.702905:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.702915:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876624.702941:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876624.702952:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-98, flags=8 00000080:00200008:1.0:1788876624.702963:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-98:[0x200001b71:0x8c:0x0](ffffa061981b80c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876624.702969:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x8c:0x0](ffffa061981b80c8),name=f5.replay-single-98 00000002:00010000:1.0:1788876624.702989:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x8c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.703011:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188690400/0x7d14dcdc37047ed8 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aae98 expref: -99 pid: 33247 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.703024:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188690400/0x7d14dcdc37047ed8 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aae98 expref: -99 pid: 33247 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.703030:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.703034:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x8c:0x0](ffffa061981b80c8) 00000080:00010000:1.0:1788876624.703037:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x8c:0x0](ffffa061981b80c8) for lock 0x7d14dcdc37047ed8 00000080:00010000:1.0:1788876624.703041:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047ed8 from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876624.703044:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188690400/0x7d14dcdc37047ed8 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aae98 expref: -99 pid: 33247 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.703052:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188690400/0x7d14dcdc37047ed8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aae98 expref: -99 pid: 33247 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.703080:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x8c:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876624.703090:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x8c:0x0] 00000008:00010000:1.0:1788876624.703110:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876624.703113:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2230, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876624.703131:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876624.703143:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 2/0,1 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876624.703155:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876624.703159:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 5/0,1 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876624.703167:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876624.703176:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876624.703189:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x8c:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876624.703193:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:1.0:1788876624.703196:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876624.703199:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2230 (ffffa06189032ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876624.703203:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876624.703206:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2230 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876624.703216:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x8c:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.703249:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.703263:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.703270:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876624.703279:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876624.703286:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.703290:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.703293:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048824 00000080:00010000:1.0:1788876624.703297:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048824 from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876624.703299:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876624.703307:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.703317:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876624.703325:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-98, flags=0 00000080:00200000:1.0:1788876624.703329:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.703335:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.703341:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.703349:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.703356:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.703360:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.703368:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048824 00000080:00010000:1.0:1788876624.703372:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048824 from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876624.703376:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.703408:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.703418:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876624.703423:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-98, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876624.703447:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.703472:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188690400/0x7d14dcdc37047ed8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aae98 expref: -99 pid: 33247 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.703482:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x8c:0x0]: losing layout lock 00010000:00010000:1.0:1788876624.703516:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188690400/0x7d14dcdc37047ed8 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aae98 expref: -99 pid: 33247 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.703524:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.703531:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876624.703534:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06188690400/0x7d14dcdc37047ed8 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x8c:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aae98 expref: -99 pid: 33247 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.703552:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048824 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae56b expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876624.704215:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876624.704226:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061887b6300 x1875771511213184/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876624.704250:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b6300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511213184:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876624.704485:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876624.715246:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061887b6300 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511213184:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876624.715304:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x8c:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876624.715317:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876624.715343:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x8c:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.715455:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8c:0x0](ffffa061981b80c8) 00000008:00010000:1.0:1788876624.715487:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 7 blocks 8 [cma]time 1788876561 1788876561 1788876617 ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 1/0,0 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876624.715560:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.715579:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.715633:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.715642:0:33371:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.715651:0:33371:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33371 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876624.715669:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b7480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511213312:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876624.715749:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876624.721304:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061887b7480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511213312:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876624.721333:0:33371:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721347:0:33371:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721361:0:33371:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721414:0:33371:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721430:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721440:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721450:0:33371:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876624.721462:0:33371:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061887b7480 x1875771511213312/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876640 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876624.721477:0:33371:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721510:0:33371:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876624.721518:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876624.721544:0:33371:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876624.721552:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704882b 00000080:00010000:1.0:1788876624.721559:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.721563:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704882b from it ffffb95ac20a7b58 00010000:00010000:1.0:1788876624.721567:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876624.721577:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876624.721606:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876624.721620:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-99, flags=8 00000080:00200008:1.0:1788876624.721633:0:33371:0:(file.c:6296:ll_getattr_dentry()) START file f5.replay-single-99:[0x200001b71:0x8d:0x0](ffffa06198248908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876624.721640:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x8d:0x0](ffffa06198248908),name=f5.replay-single-99 00000002:00010000:1.0:1788876624.721663:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x8d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.721674:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06190248200/0x7d14dcdc37047edf lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaebb expref: -99 pid: 33248 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.721686:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06190248200/0x7d14dcdc37047edf lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaebb expref: -99 pid: 33248 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.721694:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.721698:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x8d:0x0](ffffa06198248908) 00000080:00010000:1.0:1788876624.721700:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x8d:0x0](ffffa06198248908) for lock 0x7d14dcdc37047edf 00000080:00010000:1.0:1788876624.721722:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37047edf from it ffffb95ac20a7cb8 00010000:00010000:1.0:1788876624.721725:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06190248200/0x7d14dcdc37047edf lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aaebb expref: -99 pid: 33248 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.721733:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06190248200/0x7d14dcdc37047edf lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aaebb expref: -99 pid: 33248 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.721755:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x8d:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:1.0:1788876624.721765:0:33371:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x8d:0x0] 00000008:00010000:1.0:1788876624.721786:0:33371:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:1.0:1788876624.721790:0:33371:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190eb2230, osc lock ffffa06186792be0, flags 0x400001002 00010000:00010000:1.0:1788876624.721809:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876624.721821:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:1.0:1788876624.721846:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:1.0:1788876624.721851:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876624.721861:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:1.0:1788876624.721872:0:33371:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1788876624.721884:0:33371:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x8d:0x0] updating i_size 7 i_blocks 8 00000020:00010000:1.0:1788876624.721889:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:1.0:1788876624.721892:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1788876624.721903:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190eb2230 (ffffa06189032ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1788876624.721908:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1788876624.721910:0:33371:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190eb2230 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1788876624.721922:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x8d:0x0] ignore/verify layout 0/1, layout version 65535 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.721958:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.721966:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.721973:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876624.721982:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876624.721989:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.722004:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.722007:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704882b 00000080:00010000:1.0:1788876624.722011:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704882b from it ffffb95ac20a7c10 00010000:00010000:1.0:1788876624.722015:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876624.722023:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.722031:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876624.722038:0:33371:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f5.replay-single-99, flags=0 00000080:00200000:1.0:1788876624.722042:0:33371:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876624.722048:0:33371:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876624.722054:0:33371:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.722073:0:33371:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876624.722079:0:33371:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876624.722084:0:33371:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876624.722086:0:33371:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704882b 00000080:00010000:1.0:1788876624.722090:0:33371:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704882b from it ffffb95ac20a7d68 00010000:00010000:1.0:1788876624.722093:0:33371:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876624.722101:0:33371:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.722109:0:33371:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876624.722114:0:33371:0:(namei.c:2351:ll_unlink()) VFS Op:name=f5.replay-single-99, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876624.722140:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.722165:0:33371:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06190248200/0x7d14dcdc37047edf lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aaebb expref: -99 pid: 33248 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876624.722177:0:33371:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x8d:0x0]: losing layout lock 00010000:00010000:1.0:1788876624.722215:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06190248200/0x7d14dcdc37047edf lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aaebb expref: -99 pid: 33248 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.722223:0:33371:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.722241:0:33371:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876624.722245:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06190248200/0x7d14dcdc37047edf lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x8d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aaebb expref: -99 pid: 33248 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876624.722267:0:33371:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc3704882b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae595 expref: -99 pid: 33371 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876624.722286:0:33371:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876624.722294:0:33371:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061887b7480 x1875771511213568/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876624.722318:0:33371:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b7480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511213568:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876624.722430:0:33371:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64cc0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876624.726458:0:33371:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061887b7480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33371:1875771511213568:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876624.726522:0:33371:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x8d:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00020000:00200000:1.0:1788876624.726536:0:33371:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1788876624.726564:0:33371:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x8d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876624.726613:0:33371:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x8d:0x0](ffffa06198248908) 00000008:00010000:1.0:1788876624.726647:0:33371:0:(osc_object.c:202:osc_object_ast_clear()) ### update lock size 7 blocks 8 [cma]time: 1788876561 1788876561 0 by oinfo size 7 blocks 8 [cma]time 1788876561 1788876561 1788876617 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 1/0,0 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.449140:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.449175:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.449207:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511215488:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.449301:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511215744:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.449336:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216000:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.452473:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511215872:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.452957:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532254336 00000100:00100000:1.0:1788876628.452976:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532254336 00000100:00100000:1.0:1788876628.453024:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 105 00000100:00100000:1.0:1788876628.453032:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:17045:x1875771532254336:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.453058:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ceb591c3/0x7d14dcdc370478c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7cc6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.453111:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.453118:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:17045:x1875771532254336:12345-192.168.203.111@tcp:104: Request processed in 86us (2077us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.453126:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 105 00010000:00010000:1.0:1788876628.453147:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.453154:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ceb591c3/0x7d14dcdc370478c6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7cc6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.453163:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618890a400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.453168:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ceb591c3/0x7d14dcdc370478c6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7cc6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.453212:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ceb591c3/0x7d14dcdc370478c6 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7cc6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.453221:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.453239:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ceb591c3/0x7d14dcdc370478c6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7cc6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.453248:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ceb591c3/0x7d14dcdc370478c6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x21d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7cc6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.453270:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.453272:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.453281:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.453440:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511216128:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.453496:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511214336:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.454947:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511214848:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.454962:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511214464:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.454983:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511214720:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.454991:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511214976:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.454999:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511215232:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876628.456008:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.456016:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.456026:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061888d2000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.456031:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.456074:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.456082:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.456098:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.456107:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000946db6cc/0x7d14dcdc37047983 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7ebe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.456126:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.456128:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.456130:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.456142:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216384:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.463946:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532254976 00000100:00100000:1.0:1788876628.463960:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532254976 00000100:00100000:1.0:1788876628.464022:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532255104 00000100:00100000:1.0:1788876628.464027:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532255104 00000100:00100000:1.0:1788876628.464047:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 107 00000100:00100000:1.0:1788876628.464057:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198aff800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:10319:x1875771532254976:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.464077:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000063573b89/0x7d14dcdc370479ad lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7f2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.464197:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.464204:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198aff800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:10319:x1875771532254976:12345-192.168.203.111@tcp:104: Request processed in 148us (6026us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.464212:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 107 00000100:00100000:1.0:1788876628.464229:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 108 00000100:00100000:1.0:1788876628.464236:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43953:x1875771532255104:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.464246:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000006bf5a347/0x7d14dcdc3704796e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7e86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466373:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1788876628.466403:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000063573b89/0x7d14dcdc370479ad lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7f2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466416:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188824400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.466425:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000063573b89/0x7d14dcdc370479ad lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7f2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466484:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000063573b89/0x7d14dcdc370479ad lrc: 2/0,0 mode: --/PW res: [0x280000401:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7f2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466491:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.466515:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000063573b89/0x7d14dcdc370479ad lrc: 1/0,0 mode: --/PW res: [0x280000401:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7f2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466525:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000063573b89/0x7d14dcdc370479ad lrc: 0/0,0 mode: --/PW res: [0x280000401:0x222:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7f2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466545:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.466550:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000006bf5a347/0x7d14dcdc3704796e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a7e86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466557:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188995200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.466561:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000006bf5a347/0x7d14dcdc3704796e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a7e86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466589:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000006bf5a347/0x7d14dcdc3704796e lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7e86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466596:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.466603:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000006bf5a347/0x7d14dcdc3704796e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7e86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466611:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000006bf5a347/0x7d14dcdc3704796e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x221:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7e86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.466626:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.466629:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.466631:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.466653:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216640:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.466738:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1788876628.466746:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216512:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.471489:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:1.0:1788876628.471509:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43953:x1875771532255104:12345-192.168.203.111@tcp:104: Request processed in 7266us (11446us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.471522:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 108 00010000:00010000:3.0:1788876628.474091:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.475536:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511215744:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.475568:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511215104:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.475579:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511215360:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.493487:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216000:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.493501:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511214592:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.493523:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06190ea5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511215488:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.499084:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009b9e49e1/0x7d14dcdc370479c2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7f66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499098:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009b9e49e1/0x7d14dcdc370479c2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x223:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a7f66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499118:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:2.0:1788876628.499121:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d5bd3155/0x7d14dcdc37047a40 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a80b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499127:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188995c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.499131:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d5bd3155/0x7d14dcdc37047a40 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a80b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499149:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d5bd3155/0x7d14dcdc37047a40 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a80b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499156:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.499162:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d5bd3155/0x7d14dcdc37047a40 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a80b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499169:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d5bd3155/0x7d14dcdc37047a40 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a80b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499183:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1788876628.499186:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f5c492a0/0x7d14dcdc37047a55 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a80ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499198:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188994a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.499201:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f5c492a0/0x7d14dcdc37047a55 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a80ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499218:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f5c492a0/0x7d14dcdc37047a55 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a80ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499225:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.499229:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f5c492a0/0x7d14dcdc37047a55 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a80ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499237:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f5c492a0/0x7d14dcdc37047a55 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x226:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a80ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499248:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.499251:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009fef698/0x7d14dcdc37046b67 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a591e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499257:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06180030400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.499261:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009fef698/0x7d14dcdc37046b67 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a591e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499276:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009fef698/0x7d14dcdc37046b67 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a591e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499287:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.499291:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009fef698/0x7d14dcdc37046b67 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a591e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499297:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009fef698/0x7d14dcdc37046b67 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1cb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a591e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.499308:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.499310:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.499311:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.499317:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.499326:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217408:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.499350:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217920:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.499365:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876628.499370:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217280:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.499655:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188847100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217664:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.506501:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216384:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.506822:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216640:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.506914:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511216512:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.506943:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511215872:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.507484:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.507491:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000004f700092/0x7d14dcdc370469ed lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a552e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.507506:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187babc00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.507513:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000004f700092/0x7d14dcdc370469ed lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a552e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.507559:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000004f700092/0x7d14dcdc370469ed lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a552e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.507567:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.507589:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000004f700092/0x7d14dcdc370469ed lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a552e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.507596:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000004f700092/0x7d14dcdc370469ed lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1c2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a552e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.507615:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.507617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.507619:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.507628:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1788876628.507641:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188846300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511218048:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.511321:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1788876628.511329:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d4862801/0x7d14dcdc37047a6a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8126 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511337:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188825800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.511344:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d4862801/0x7d14dcdc37047a6a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a8126 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511475:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d4862801/0x7d14dcdc37047a6a lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8126 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511485:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.511506:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d4862801/0x7d14dcdc37047a6a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8126 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511514:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d4862801/0x7d14dcdc37047a6a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8126 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511534:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1788876628.511537:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008d0d42a4/0x7d14dcdc37047a94 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8196 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511544:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061887ad800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.511548:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008d0d42a4/0x7d14dcdc37047a94 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a8196 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511570:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008d0d42a4/0x7d14dcdc37047a94 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8196 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511577:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.511583:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008d0d42a4/0x7d14dcdc37047a94 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8196 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.511584:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1788876628.511590:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000aa149ac/0x7d14dcdc37047a7f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a815e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511591:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008d0d42a4/0x7d14dcdc37047a94 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8196 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.511598:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188824c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.511602:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000aa149ac/0x7d14dcdc37047a7f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a815e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511606:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.511609:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008ba8340d/0x7d14dcdc37047aa9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a81ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511615:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188994e00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.511618:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008ba8340d/0x7d14dcdc37047aa9 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a81ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.511626:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000aa149ac/0x7d14dcdc37047a7f lrc: 2/0,0 mode: --/PW res: [0x280000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a815e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.511633:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.511634:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008ba8340d/0x7d14dcdc37047aa9 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a81ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.511638:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000aa149ac/0x7d14dcdc37047a7f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a815e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511640:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.511644:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008ba8340d/0x7d14dcdc37047aa9 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a81ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.511646:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000aa149ac/0x7d14dcdc37047a7f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x227:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a815e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.511652:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008ba8340d/0x7d14dcdc37047aa9 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x228:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a81ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.511660:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.511661:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.511662:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.511663:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.511663:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.511665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.512475:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188844e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218304:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.512536:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218560:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.512563:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1788876628.512569:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188845880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218176:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.512590:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218432:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.512619:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.514475:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1788876628.514483:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d193f1ab/0x7d14dcdc37047ad3 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a823e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.514497:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188994800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.514503:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d193f1ab/0x7d14dcdc37047ad3 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a823e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.514543:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d193f1ab/0x7d14dcdc37047ad3 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a823e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.514561:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.514567:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.514573:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b4c70ac4/0x7d14dcdc37047ae8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8276 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.514578:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d193f1ab/0x7d14dcdc37047ad3 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a823e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.514580:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618897a800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.514584:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b4c70ac4/0x7d14dcdc37047ae8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a8276 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.514587:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d193f1ab/0x7d14dcdc37047ad3 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a823e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.514607:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b4c70ac4/0x7d14dcdc37047ae8 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8276 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.514607:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.514609:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.514611:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.514614:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.514620:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b4c70ac4/0x7d14dcdc37047ae8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8276 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.514627:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b4c70ac4/0x7d14dcdc37047ae8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8276 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.514641:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.514643:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.514644:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.514658:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218816:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.514773:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788876628.514782:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218688:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.523293:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.523468:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:2.0:1788876628.523509:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188847800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217408:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.523524:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217920:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.523534:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217152:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.523538:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.523738:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:3.0:1788876628.526886:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532259968 00000100:00100000:3.0:1788876628.526901:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532259968 00000100:00100000:3.0:1788876628.526961:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532260352 00000100:00100000:3.0:1788876628.526973:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532260352 00000100:00100000:3.0:1788876628.526997:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 14 00000100:00100000:3.0:1788876628.527007:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:43955:x1875771532259968:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.527028:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ea162260/0x7d14dcdc37047afd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a82ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527129:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1788876628.527136:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:43955:x1875771532259968:12345-192.168.203.111@tcp:104: Request processed in 131us (3668us total) trans 0 rc 0/0 00000100:00100000:3.0:1788876628.527144:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 14 00000100:00100000:3.0:1788876628.527164:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 15 00000100:00100000:3.0:1788876628.527172:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:43959:x1875771532260352:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.527181:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527210:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:3.0:1788876628.527216:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:43959:x1875771532260352:12345-192.168.203.111@tcp:104: Request processed in 46us (3678us total) trans 0 rc 0/0 00000100:00100000:3.0:1788876628.527223:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 15 00010000:00010000:3.0:1788876628.527245:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1788876628.527252:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ea162260/0x7d14dcdc37047afd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a82ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527261:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618897ae00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.527267:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ea162260/0x7d14dcdc37047afd lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a82ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527315:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ea162260/0x7d14dcdc37047afd lrc: 2/0,0 mode: --/PW res: [0x280000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a82ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527324:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.527347:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ea162260/0x7d14dcdc37047afd lrc: 1/0,0 mode: --/PW res: [0x280000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a82ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527357:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ea162260/0x7d14dcdc37047afd lrc: 0/0,0 mode: --/PW res: [0x280000401:0x22a:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a82ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527397:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.527401:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527408:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618897a400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.527412:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527430:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527438:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.527444:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527451:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c834a981/0x7d14dcdc37047b27 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a831e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.527465:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.527467:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.527469:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.527485:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511219072:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.527558:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1788876628.527565:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218944:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876628.527596:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.530558:0:15181:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876628.530591:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532259712 00000100:00100000:1.0:1788876628.530600:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532259712 00000100:00100000:1.0:1788876628.530649:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 125 00000100:00100000:1.0:1788876628.530658:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06186253050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43953:x1875771532259712:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.530677:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.530743:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.530750:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06186253050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43953:x1875771532259712:12345-192.168.203.111@tcp:104: Request processed in 93us (7466us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.530757:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 125 00000100:00100000:1.0:1788876628.530841:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511217536:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.530855:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511217792:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.530898:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.530905:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.530914:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188979200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.530919:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.530960:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.530977:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.530996:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531010:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000067d76be4/0x7d14dcdc37047abe lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x229:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8206 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531030:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.531032:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.531034:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.531048:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511219200:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.531131:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.531172:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532260992 00000100:00100000:1.0:1788876628.531175:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532260992 00000100:00100000:1.0:1788876628.531195:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 126 00000100:00100000:1.0:1788876628.531200:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532260992:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.531209:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531237:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.531242:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b7800 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532260992:12345-192.168.203.111@tcp:104: Request processed in 41us (113us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.531248:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 126 00010000:00010000:1.0:1788876628.531262:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.531267:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531274:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618897b600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.531278:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531305:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531312:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.531323:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531331:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a5bd5256/0x7d14dcdc37047b3c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x22c:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8356 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.531345:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.531346:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.531348:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.531356:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511219328:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.532623:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532260736 00000100:00100000:3.0:1788876628.532633:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532260736 00000100:00100000:3.0:1788876628.532677:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 16 00000100:00100000:3.0:1788876628.532747:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:43962:x1875771532260736:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.532759:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.532909:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1788876628.532916:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b7100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:43962:x1875771532260736:12345-192.168.203.111@tcp:104: Request processed in 172us (9178us total) trans 0 rc 0/0 00000100:00100000:3.0:1788876628.536742:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 16 00010000:00010000:3.0:1788876628.536774:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.536784:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.536798:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188978800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.536906:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.536974:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.536983:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.537043:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.537053:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e90ed168/0x7d14dcdc37047b12 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x22b:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a82e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:2.0:1788876628.537063:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511219456:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876628.537074:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.537076:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.537078:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.558636:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:3.0:1788876628.559071:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188844e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218304:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.562889:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.563944:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.563998:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.564031:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532261248 00000100:00100000:1.0:1788876628.564039:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532261248 00000100:00100000:1.0:1788876628.564075:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532261632 00000100:00100000:1.0:1788876628.564077:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532261632 00000100:00100000:1.0:1788876628.564086:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532261760 00000100:00100000:1.0:1788876628.564088:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532261760 00000100:00100000:1.0:1788876628.564102:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 128 00000100:00100000:1.0:1788876628.564111:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43960:x1875771532261248:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.564142:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b94f7be8/0x7d14dcdc37046b7c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5956 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.564218:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.564225:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b5f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43960:x1875771532261248:12345-192.168.203.111@tcp:104: Request processed in 115us (1346us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.564233:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 128 00000100:00100000:1.0:1788876628.564250:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 129 00000100:00100000:1.0:1788876628.564256:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:43964:x1875771532261632:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.564266:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003813ddfb/0x7d14dcdc37047ba5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a846e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.564291:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876628.564296:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+6:43964:x1875771532261632:12345-192.168.203.111@tcp:104: Request processed in 40us (356us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.564302:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 129 00000100:00100000:1.0:1788876628.564313:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 130 00000100:00100000:1.0:1788876628.564319:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:43952:x1875771532261760:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.564326:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001e46cf43/0x7d14dcdc37047b7b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a83fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.564351:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876628.564357:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b4380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+5:43952:x1875771532261760:12345-192.168.203.111@tcp:104: Request processed in 38us (359us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.564362:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 130 00000100:00100000:3.0:1788876628.564761:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218560:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.564849:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218944:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.564866:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511219072:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.564876:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218688:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.564886:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218816:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.564896:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188845880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218176:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.564905:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511218432:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.568806:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061887b5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511219328:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.568851:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.568860:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b94f7be8/0x7d14dcdc37046b7c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5956 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.568872:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06180031c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.568879:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b94f7be8/0x7d14dcdc37046b7c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5956 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.568930:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b94f7be8/0x7d14dcdc37046b7c lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5956 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.568939:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.568956:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b94f7be8/0x7d14dcdc37046b7c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5956 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569231:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b94f7be8/0x7d14dcdc37046b7c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5956 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569253:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.569258:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003813ddfb/0x7d14dcdc37047ba5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a846e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569265:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061888f8200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.569269:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003813ddfb/0x7d14dcdc37047ba5 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a846e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569293:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003813ddfb/0x7d14dcdc37047ba5 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a846e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569306:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.569314:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003813ddfb/0x7d14dcdc37047ba5 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a846e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569322:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003813ddfb/0x7d14dcdc37047ba5 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a846e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569337:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.569341:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001e46cf43/0x7d14dcdc37047b7b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a83fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.569347:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618897a200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.569351:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001e46cf43/0x7d14dcdc37047b7b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a83fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:2.0:1788876628.570113:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511218048:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.570127:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188846a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217280:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.570135:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188847100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511217664:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.570172:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511219456:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.570337:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001e46cf43/0x7d14dcdc37047b7b lrc: 2/0,0 mode: --/PW res: [0x280000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a83fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.570344:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.570350:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001e46cf43/0x7d14dcdc37047b7b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a83fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.570368:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001e46cf43/0x7d14dcdc37047b7b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a83fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.570379:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.570556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.570557:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.570576:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532262016 00000100:00100000:1.0:1788876628.570582:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532262016 00000100:00100000:1.0:1788876628.570621:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 131 00000100:00100000:1.0:1788876628.570628:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198affb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43954:x1875771532262016:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.570638:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003e11d99c/0x7d14dcdc37046b91 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a598e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.572310:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.572325:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198affb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43954:x1875771532262016:12345-192.168.203.111@tcp:104: Request processed in 1693us (1789us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.572336:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 131 00010000:00010000:1.0:1788876628.572921:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.572941:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003e11d99c/0x7d14dcdc37046b91 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a598e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.572955:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b74600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.572975:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003e11d99c/0x7d14dcdc37046b91 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a598e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573031:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003e11d99c/0x7d14dcdc37046b91 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a598e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573038:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.573064:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003e11d99c/0x7d14dcdc37046b91 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a598e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573084:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000003e11d99c/0x7d14dcdc37046b91 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1cc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a598e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573109:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.573111:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.573113:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.573121:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788876628.573137:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511220096:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.573261:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.573292:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.573314:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.573345:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:3.0:1788876628.573347:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532263168 00000100:00100000:3.0:1788876628.573355:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532263168 00000100:00100000:1.0:1788876628.573359:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.573407:0:15182:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1788876628.573423:0:15183:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:3.0:1788876628.573440:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532263296 00000100:00100000:3.0:1788876628.573443:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532263296 00000100:00100000:3.0:1788876628.573456:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532263424 00000100:00100000:3.0:1788876628.573459:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532263424 00000100:00100000:3.0:1788876628.573468:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532263552 00000100:00100000:3.0:1788876628.573470:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532263552 00000100:00100000:3.0:1788876628.573482:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 17 00000100:00100000:3.0:1788876628.573489:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:43956:x1875771532263168:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.573500:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.573536:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.573538:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532262784 00000100:00100000:1.0:1788876628.573541:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532262784 00000100:00100000:3.0:1788876628.573542:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b5880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:43956:x1875771532263168:12345-192.168.203.111@tcp:104: Request processed in 53us (249us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.573565:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 132 00000100:00100000:1.0:1788876628.573571:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1fab050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8453:x1875771532262784:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.573580:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fe43d95b/0x7d14dcdc37047bcf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a84de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573604:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876628.573609:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1fab050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8453:x1875771532262784:12345-192.168.203.111@tcp:104: Request processed in 39us (350us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.573616:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 132 00010000:00010000:1.0:1788876628.573654:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.573658:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573732:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061889ada00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.573737:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573764:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573770:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.573781:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573788:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bee35048/0x7d14dcdc37047bba lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a84a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573803:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.573807:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fe43d95b/0x7d14dcdc37047bcf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a84de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573813:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061889ac400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.573816:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fe43d95b/0x7d14dcdc37047bcf lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a84de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573833:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fe43d95b/0x7d14dcdc37047bcf lrc: 2/0,0 mode: --/PW res: [0x280000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a84de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573844:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.573852:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fe43d95b/0x7d14dcdc37047bcf lrc: 1/0,0 mode: --/PW res: [0x280000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a84de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573859:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000fe43d95b/0x7d14dcdc37047bcf lrc: 0/0,0 mode: --/PW res: [0x280000401:0x22f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a84de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.573871:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.573872:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.573874:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.573886:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511220224:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.573913:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788876628.573918:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511220352:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.573956:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:1.0:1788876628.574021:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.575394:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 17 00000100:00100000:3.0:1788876628.575420:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532263680 00000100:00100000:3.0:1788876628.575425:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532263680 00000100:00100000:3.0:1788876628.575452:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 18 00000100:00100000:3.0:1788876628.575460:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:43959:x1875771532263296:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.575473:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000056933c48/0x7d14dcdc37047b90 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8436 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.575519:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1788876628.575526:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b4700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:43959:x1875771532263296:12345-192.168.203.111@tcp:104: Request processed in 67us (2210us total) trans 0 rc 0/0 00000100:00100000:3.0:1788876628.575532:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 18 00010000:00010000:1.0:1788876628.576376:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.576396:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000056933c48/0x7d14dcdc37047b90 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a8436 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.576407:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618897ba00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.576413:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000056933c48/0x7d14dcdc37047b90 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a8436 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.576458:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000056933c48/0x7d14dcdc37047b90 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8436 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.576481:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.576502:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000056933c48/0x7d14dcdc37047b90 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8436 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.576533:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000056933c48/0x7d14dcdc37047b90 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x22e:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a8436 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.576561:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.576563:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.576564:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.576581:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511220480:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.576696:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 19 00000100:00100000:3.0:1788876628.576707:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:43955:x1875771532263424:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.576717:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007e9785a3/0x7d14dcdc37046bfa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5aa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.576774:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:3.0:1788876628.576780:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b6d80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:43955:x1875771532263424:12345-192.168.203.111@tcp:104: Request processed in 77us (3435us total) trans 0 rc 0/0 00000100:00100000:3.0:1788876628.576974:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 19 00000100:00100000:3.0:1788876628.576996:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 20 00000100:00100000:3.0:1788876628.577003:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1e06050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:8456:x1875771532263552:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.577012:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.577041:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:3.0:1788876628.577046:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1e06050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+5:8456:x1875771532263552:12345-192.168.203.111@tcp:104: Request processed in 44us (3686us total) trans 0 rc 0/0 00000100:00100000:3.0:1788876628.577052:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 20 00000100:00100000:3.0:1788876628.577081:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 21 00000100:00100000:3.0:1788876628.577088:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:43953:x1875771532263680:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.577096:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c790d218/0x7d14dcdc37046ba6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a59c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.577120:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:3.0:1788876628.577125:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b5c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+4:43953:x1875771532263680:12345-192.168.203.111@tcp:104: Request processed in 37us (3168us total) trans 0 rc 0/0 00000100:00100000:3.0:1788876628.577130:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 21 00000100:00100000:1.0:1788876628.577302:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.577324:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.577338:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.577435:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:1.0:1788876628.577451:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.577456:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007e9785a3/0x7d14dcdc37046bfa lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5aa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577464:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b74e00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.577468:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007e9785a3/0x7d14dcdc37046bfa lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5aa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577498:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007e9785a3/0x7d14dcdc37046bfa lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5aa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577516:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.577614:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007e9785a3/0x7d14dcdc37046bfa lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5aa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577622:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007e9785a3/0x7d14dcdc37046bfa lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5aa6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577640:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.577644:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577650:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b75400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.577653:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577670:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577677:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.577683:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577689:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fd4647e4/0x7d14dcdc37046bd0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5a36 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577701:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.577710:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c790d218/0x7d14dcdc37046ba6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a59c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.577717:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b75a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.577721:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c790d218/0x7d14dcdc37046ba6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a59c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.579667:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c790d218/0x7d14dcdc37046ba6 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a59c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.579685:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.579699:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c790d218/0x7d14dcdc37046ba6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a59c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.579707:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c790d218/0x7d14dcdc37046ba6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a59c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.579733:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.579735:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.579738:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.579768:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511220736:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.579906:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788876628.579914:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511220608:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.579933:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061887b4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511220864:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.579977:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532264192 00000100:00100000:1.0:1788876628.579984:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532264192 00000100:00100000:1.0:1788876628.580032:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532264320 00000100:00100000:1.0:1788876628.580035:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532264320 00000100:00100000:1.0:1788876628.580046:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532264448 00000100:00100000:1.0:1788876628.580048:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532264448 00000100:00100000:1.0:1788876628.580058:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532264576 00000100:00100000:1.0:1788876628.580061:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532264576 00000100:00100000:1.0:1788876628.580073:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 133 00000100:00100000:1.0:1788876628.580080:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43963:x1875771532264192:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.580094:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000057d4d37/0x7d14dcdc37047b66 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a83c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580135:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.580141:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b5180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43963:x1875771532264192:12345-192.168.203.111@tcp:104: Request processed in 62us (2840us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.580147:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 133 00000100:00100000:1.0:1788876628.580163:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 134 00000100:00100000:1.0:1788876628.580169:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43962:x1875771532264320:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.580177:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d85d8f00/0x7d14dcdc37046be5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5a6e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580197:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876628.580202:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b7b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43962:x1875771532264320:12345-192.168.203.111@tcp:104: Request processed in 33us (2876us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.580207:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 134 00000100:00100000:1.0:1788876628.580217:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 135 00000100:00100000:1.0:1788876628.580222:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:10319:x1875771532264448:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.580229:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000052c2b506/0x7d14dcdc37046c0f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5ade expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580253:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876628.580257:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b4000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:10319:x1875771532264448:12345-192.168.203.111@tcp:104: Request processed in 36us (2918us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.580263:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 135 00000100:00100000:1.0:1788876628.580274:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 136 00000100:00100000:1.0:1788876628.580279:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061887b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532264576:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.580287:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000044ddeeaf/0x7d14dcdc37046bbb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a59fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580305:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1788876628.580309:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061887b6a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532264576:12345-192.168.203.111@tcp:104: Request processed in 31us (2875us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.580314:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 136 00010000:00010000:1.0:1788876628.580326:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1788876628.580331:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000057d4d37/0x7d14dcdc37047b66 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a83c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580339:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618897b200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.580345:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000057d4d37/0x7d14dcdc37047b66 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a83c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580398:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000057d4d37/0x7d14dcdc37047b66 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a83c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580407:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.580425:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000057d4d37/0x7d14dcdc37047b66 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a83c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580432:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000057d4d37/0x7d14dcdc37047b66 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x22d:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a83c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580456:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.580460:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d85d8f00/0x7d14dcdc37046be5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5a6e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580466:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b74000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.580470:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d85d8f00/0x7d14dcdc37046be5 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5a6e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580493:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d85d8f00/0x7d14dcdc37046be5 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5a6e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580500:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.580507:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d85d8f00/0x7d14dcdc37046be5 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5a6e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580514:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d85d8f00/0x7d14dcdc37046be5 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1ce:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5a6e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.580526:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.580529:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000052c2b506/0x7d14dcdc37046c0f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5ade expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.582757:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b74c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.582766:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000052c2b506/0x7d14dcdc37046c0f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5ade expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.583537:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000052c2b506/0x7d14dcdc37046c0f lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5ade expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.583744:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.583756:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000052c2b506/0x7d14dcdc37046c0f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5ade expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.583764:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000052c2b506/0x7d14dcdc37046c0f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1cf:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5ade expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.583787:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.583793:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000044ddeeaf/0x7d14dcdc37046bbb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a59fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.583800:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b74800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.584977:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000044ddeeaf/0x7d14dcdc37046bbb lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a59fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.585007:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000044ddeeaf/0x7d14dcdc37046bbb lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a59fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.585025:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.585049:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000044ddeeaf/0x7d14dcdc37046bbb lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a59fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.585057:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000044ddeeaf/0x7d14dcdc37046bbb lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1cd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a59fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.585072:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.585074:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.585076:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.585085:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788876628.585095:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190289500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511221504:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.597650:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.597658:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c73abed2/0x7d14dcdc37046a02 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5566 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.597679:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061890a0600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.597686:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c73abed2/0x7d14dcdc37046a02 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5566 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.597731:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c73abed2/0x7d14dcdc37046a02 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5566 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.597739:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.597760:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c73abed2/0x7d14dcdc37046a02 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5566 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.597768:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000c73abed2/0x7d14dcdc37046a02 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5566 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.597789:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.597791:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.597793:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.597799:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1788876628.597811:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511221632:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.610123:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610161:0:15181:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876628.610180:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532266112 00000100:00100000:1.0:1788876628.610189:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532266112 00000100:00100000:1.0:1788876628.610238:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 139 00000100:00100000:1.0:1788876628.610248:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1fa8850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43961:x1875771532266112:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.610268:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000169be809/0x7d14dcdc37046c4e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5b86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.610347:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.610354:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1fa8850 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43961:x1875771532266112:12345-192.168.203.111@tcp:104: Request processed in 108us (242us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.610362:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 139 00010000:00010000:2.0:1788876628.610370:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.610377:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000169be809/0x7d14dcdc37046c4e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5b86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.610414:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187bab800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.610420:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000169be809/0x7d14dcdc37046c4e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5b86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.610462:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:2.0:1788876628.610465:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000169be809/0x7d14dcdc37046c4e lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5b86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.610472:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.610490:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000169be809/0x7d14dcdc37046c4e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5b86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.610499:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000169be809/0x7d14dcdc37046c4e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5b86 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.610521:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.610522:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.610524:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.610540:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511221760:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.610575:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610623:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610640:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610651:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610663:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610676:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610691:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610736:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:2.0:1788876628.610748:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532267648 00000100:00100000:2.0:1788876628.610752:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532267648 00000100:00100000:1.0:1788876628.610755:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.610766:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:2.0:1788876628.610771:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532268032 00000100:00100000:2.0:1788876628.610774:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532268032 00000100:00100000:1.0:1788876628.610780:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:2.0:1788876628.610782:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532268160 00000100:00100000:2.0:1788876628.610784:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532268160 00000100:00100000:2.0:1788876628.610793:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532268288 00000100:00100000:2.0:1788876628.610795:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532268288 00000100:00100000:1.0:1788876628.610796:0:33373:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876628.610803:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532266240 00000100:00100000:2.0:1788876628.610803:0:15183:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532268416 00000100:00100000:1.0:1788876628.610805:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532266240 00000100:00100000:2.0:1788876628.610805:0:15183:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532268416 00000100:00100000:2.0:1788876628.610815:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 22 00000100:00100000:1.0:1788876628.610816:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532266368 00000100:00100000:1.0:1788876628.610818:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532266368 00000100:00100000:2.0:1788876628.610821:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0619028a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+8:43955:x1875771532267648:12345-192.168.203.111@tcp:104: 00000100:00100000:1.0:1788876628.610826:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532266496 00000100:00100000:1.0:1788876628.610828:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532266496 00010000:00010000:2.0:1788876628.610831:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000883a49a/0x7d14dcdc37046c63 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5bbe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.610835:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532266624 00000100:00100000:1.0:1788876628.610837:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532266624 00000100:00100000:1.0:1788876628.610844:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532266752 00000100:00100000:1.0:1788876628.610846:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532266752 00000100:00100000:1.0:1788876628.610854:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532266880 00000100:00100000:1.0:1788876628.610856:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532266880 00000100:00100000:1.0:1788876628.610865:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532267264 00010000:00010000:2.0:1788876628.610865:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.610867:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532267264 00000100:00100000:2.0:1788876628.610870:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0619028a680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+9:43955:x1875771532267648:12345-192.168.203.111@tcp:104: Request processed in 49us (178us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876628.610876:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 22 00000100:00100000:1.0:1788876628.610877:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 140 00000100:00100000:1.0:1788876628.610882:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190289180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+9:8455:x1875771532266240:12345-192.168.203.111@tcp:104: 00000100:00100000:2.0:1788876628.610887:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 23 00010000:00010000:1.0:1788876628.610891:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9e3712d/0x7d14dcdc37046c78 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5bf6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:2.0:1788876628.610892:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190289880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+8:8456:x1875771532268032:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876628.610899:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005093816c/0x7d14dcdc37046c8d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5c2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.610925:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:2.0:1788876628.610930:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190289880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+8:8456:x1875771532268032:12345-192.168.203.111@tcp:104: Request processed in 38us (194us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876628.610936:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 23 00000100:00100000:2.0:1788876628.610947:0:15183:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 24 00000100:00100000:2.0:1788876628.610953:0:15183:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190288700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+7:43954:x1875771532268160:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876628.610962:0:15183:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.613953:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:1.0:1788876628.613981:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190289180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+9:8455:x1875771532266240:12345-192.168.203.111@tcp:104: Request processed in 3082us (3503us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.613993:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 140 00000100:00100000:1.0:1788876628.614030:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 142 00000100:00100000:1.0:1788876628.614043:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190289c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:43963:x1875771532266496:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.614061:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008a60e395/0x7d14dcdc37046cf6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5d46 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614123:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:1.0:1788876628.614129:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190289c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:43963:x1875771532266496:12345-192.168.203.111@tcp:104: Request processed in 92us (3506us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.614135:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 142 00000100:00100000:1.0:1788876628.614148:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 143 00000100:00100000:1.0:1788876628.614155:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190288380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:43962:x1875771532266624:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.614164:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614187:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/7 locks to priority blp list, 7 blwis in pool 00000100:00100000:1.0:1788876628.614192:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190288380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:43962:x1875771532266624:12345-192.168.203.111@tcp:104: Request processed in 38us (3551us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.614198:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 143 00000100:00100000:1.0:1788876628.614210:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 144 00000100:00100000:1.0:1788876628.614216:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190288e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43956:x1875771532266752:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.614240:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614264:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/8 locks to priority blp list, 8 blwis in pool 00000100:00100000:1.0:1788876628.614269:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190288e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43956:x1875771532266752:12345-192.168.203.111@tcp:104: Request processed in 53us (3617us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.614276:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 144 00000100:00100000:1.0:1788876628.614288:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 145 00000100:00100000:1.0:1788876628.614294:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06190288a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43958:x1875771532266880:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.614303:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614320:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/9 locks to priority blp list, 9 blwis in pool 00000100:00100000:1.0:1788876628.614325:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190288a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43958:x1875771532266880:12345-192.168.203.111@tcp:104: Request processed in 32us (3662us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.614331:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 145 00000100:00100000:1.0:1788876628.614342:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 146 00000100:00100000:1.0:1788876628.614349:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1fa9450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:43953:x1875771532267264:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.614358:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000060d3e788/0x7d14dcdc37046c39 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5b4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614376:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/10 locks to priority blp list, 10 blwis in pool 00000100:00100000:1.0:1788876628.614398:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1fa9450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:43953:x1875771532267264:12345-192.168.203.111@tcp:104: Request processed in 33us (3705us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.614404:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 146 00010000:00010000:1.0:1788876628.614424:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 9 total in blp. (9 blwis in pool) 00010000:00010000:1.0:1788876628.614436:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000883a49a/0x7d14dcdc37046c63 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5bbe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614445:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061813a1a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.614451:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000883a49a/0x7d14dcdc37046c63 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5bbe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614496:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000883a49a/0x7d14dcdc37046c63 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5bbe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614504:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.614534:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000883a49a/0x7d14dcdc37046c63 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5bbe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.614542:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000883a49a/0x7d14dcdc37046c63 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1d1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5bbe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616563:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 8 total in blp. (8 blwis in pool) 00010000:00010000:1.0:1788876628.616570:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005093816c/0x7d14dcdc37046c8d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5c2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616580:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188757800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.616586:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005093816c/0x7d14dcdc37046c8d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5c2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616633:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005093816c/0x7d14dcdc37046c8d lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616657:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.616671:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005093816c/0x7d14dcdc37046c8d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616679:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005093816c/0x7d14dcdc37046c8d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616694:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 7 total in blp. (7 blwis in pool) 00010000:00010000:1.0:1788876628.616698:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9e3712d/0x7d14dcdc37046c78 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5bf6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616704:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187b75200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.616708:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9e3712d/0x7d14dcdc37046c78 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5bf6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616730:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9e3712d/0x7d14dcdc37046c78 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5bf6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616736:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.616741:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9e3712d/0x7d14dcdc37046c78 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5bf6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616748:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9e3712d/0x7d14dcdc37046c78 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5bf6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616761:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:1.0:1788876628.616770:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616777:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188756200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.616781:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616817:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616824:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.616829:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616835:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d596b556/0x7d14dcdc37046cb7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c9e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616848:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:1.0:1788876628.616852:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003bbf85ef/0x7d14dcdc37046ca2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5c66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616858:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188757000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.616861:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003bbf85ef/0x7d14dcdc37046ca2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5c66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616880:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003bbf85ef/0x7d14dcdc37046ca2 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616896:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.616901:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003bbf85ef/0x7d14dcdc37046ca2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616907:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000003bbf85ef/0x7d14dcdc37046ca2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5c66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616919:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:1.0:1788876628.616923:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008a60e395/0x7d14dcdc37046cf6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5d46 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616929:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06180031e00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.616932:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008a60e395/0x7d14dcdc37046cf6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5d46 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616950:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008a60e395/0x7d14dcdc37046cf6 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d46 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616957:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.616961:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008a60e395/0x7d14dcdc37046cf6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d46 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616976:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008a60e395/0x7d14dcdc37046cf6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d46 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.616991:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1788876628.617002:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617012:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188756a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.617017:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617036:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617043:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.617048:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617055:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fb421757/0x7d14dcdc37046d20 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5db6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617068:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.617071:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617078:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188757600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.617081:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617100:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617115:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.617145:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617152:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000760f4247/0x7d14dcdc37046d4a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5e26 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617166:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.617170:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617175:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061813a0a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.617179:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617198:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617206:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.617210:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617217:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005733cc18/0x7d14dcdc37046ccc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5cd6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617230:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.617239:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000060d3e788/0x7d14dcdc37046c39 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5b4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617245:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187baac00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.617249:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000060d3e788/0x7d14dcdc37046c39 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5b4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617268:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000060d3e788/0x7d14dcdc37046c39 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5b4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617274:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.617279:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000060d3e788/0x7d14dcdc37046c39 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5b4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617285:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000060d3e788/0x7d14dcdc37046c39 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1d0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5b4e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.617296:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.617298:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.617301:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.617404:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.617607:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0619028bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511221888:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617688:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0619028b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222144:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617710:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0619028b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222400:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617730:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190289500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222656:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617764:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222912:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617788:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 5 async RPCs [0->1] 00000100:00100000:1.0:1788876628.617794:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06190288000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222016:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617815:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0619028ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222272:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617834:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0619028aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222528:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617850:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511222784:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.617865:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511223040:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.619175:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.619224:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.619237:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.619466:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.619489:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532268800 00000100:00100000:1.0:1788876628.619497:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532268800 00000100:00100000:1.0:1788876628.619531:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532268928 00000100:00100000:1.0:1788876628.619534:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532268928 00000100:00100000:1.0:1788876628.619543:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532269184 00000100:00100000:1.0:1788876628.619545:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532269184 00000100:00100000:1.0:1788876628.619621:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532269440 00000100:00100000:1.0:1788876628.619625:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532269440 00000100:00100000:1.0:1788876628.619640:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 147 00000100:00100000:1.0:1788876628.619649:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188699f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:43960:x1875771532268800:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.619666:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d3549a53/0x7d14dcdc37046d0b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5d7e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.619788:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.619795:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188699f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+8:43960:x1875771532268800:12345-192.168.203.111@tcp:104: Request processed in 146us (625us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.619901:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 147 00000100:00100000:1.0:1788876628.619920:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 148 00000100:00100000:1.0:1788876628.619927:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188699c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:43966:x1875771532268928:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.619936:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.619982:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876628.619988:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188699c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+7:43966:x1875771532268928:12345-192.168.203.111@tcp:104: Request processed in 62us (764us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.619994:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 148 00000100:00100000:1.0:1788876628.620007:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 149 00000100:00100000:1.0:1788876628.620013:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:17045:x1875771532269184:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.620022:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620044:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876628.620049:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:17045:x1875771532269184:12345-192.168.203.111@tcp:104: Request processed in 37us (812us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.620055:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 149 00000100:00100000:1.0:1788876628.620065:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 150 00000100:00100000:1.0:1788876628.620070:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43964:x1875771532269440:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.620086:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000000722184d/0x7d14dcdc37046d9e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5f06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620109:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1788876628.620114:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43964:x1875771532269440:12345-192.168.203.111@tcp:104: Request processed in 44us (649us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.620121:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 150 00010000:00010000:1.0:1788876628.620137:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1788876628.620143:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d3549a53/0x7d14dcdc37046d0b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5d7e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620152:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188757c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.620158:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d3549a53/0x7d14dcdc37046d0b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5d7e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620196:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d3549a53/0x7d14dcdc37046d0b lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d7e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620205:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.620220:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d3549a53/0x7d14dcdc37046d0b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d7e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620229:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d3549a53/0x7d14dcdc37046d0b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1d5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d7e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620250:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.620254:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620267:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188756400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.620271:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620290:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620297:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.620303:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620310:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000072391d64/0x7d14dcdc37046d35 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1d6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5dee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620323:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.620326:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620332:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061883ace00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.620335:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620352:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620372:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.620379:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620408:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000001be36394/0x7d14dcdc37046d74 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5e96 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620421:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.620424:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000000722184d/0x7d14dcdc37046d9e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5f06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620430:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886ce400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.620434:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000000722184d/0x7d14dcdc37046d9e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5f06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620452:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000000722184d/0x7d14dcdc37046d9e lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5f06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620458:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.620463:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000000722184d/0x7d14dcdc37046d9e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5f06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620471:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000000722184d/0x7d14dcdc37046d9e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1d9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5f06 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.620484:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.620491:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.620493:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.620500:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.621499:0:15183:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:2.0:1788876628.621514:0:15183:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06190288700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_000:LOV_OSC_UUID+6:43954:x1875771532268160:12345-192.168.203.111@tcp:104: Request processed in 10556us (10752us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876628.621523:0:15183:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 24 00000100:00100000:2.0:1788876628.621585:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511221760:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.621605:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511221632:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.621640:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 25 00000100:00100000:2.0:1788876628.621650:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0619028a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:8453:x1875771532268288:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876628.621662:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.621719:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:2.0:1788876628.621725:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0619028a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:8453:x1875771532268288:12345-192.168.203.111@tcp:104: Request processed in 78us (10958us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876628.621735:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 25 00000100:00100000:2.0:1788876628.621749:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 26 00000100:00100000:2.0:1788876628.621756:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0619028b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:43952:x1875771532268416:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876628.621765:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000de80f490/0x7d14dcdc37046a17 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a559e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.621788:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1788876628.621793:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0619028b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:43952:x1875771532268416:12345-192.168.203.111@tcp:104: Request processed in 39us (11013us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876628.621799:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 26 00010000:00010000:2.0:1788876628.621815:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876628.621820:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.621828:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061883ad400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.621832:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.621861:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.621869:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.621887:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.621895:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000029a7cf24/0x7d14dcdc37046ce1 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1d4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5d0e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.621911:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.621914:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000de80f490/0x7d14dcdc37046a17 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a559e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.621920:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187baba00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.621924:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000de80f490/0x7d14dcdc37046a17 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a559e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.621942:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000de80f490/0x7d14dcdc37046a17 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a559e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.621949:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.621955:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000de80f490/0x7d14dcdc37046a17 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a559e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.621975:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000de80f490/0x7d14dcdc37046a17 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1c3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a559e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1788876628.621987:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.621989:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.621990:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.622005:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511223680:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.622048:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788876628.622054:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511223808:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.624471:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.624491:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532269952 00000100:00100000:1.0:1788876628.624499:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532269952 00000100:00100000:1.0:1788876628.624539:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 151 00000100:00100000:1.0:1788876628.624560:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43961:x1875771532269952:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.624577:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a2a6970f/0x7d14dcdc37046df2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5fe6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.624618:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.624624:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43961:x1875771532269952:12345-192.168.203.111@tcp:104: Request processed in 78us (157us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.624631:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 151 00010000:00010000:1.0:1788876628.624649:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.624655:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a2a6970f/0x7d14dcdc37046df2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5fe6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.624665:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886cec00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.624670:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a2a6970f/0x7d14dcdc37046df2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5fe6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.624718:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a2a6970f/0x7d14dcdc37046df2 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5fe6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.624727:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.624745:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a2a6970f/0x7d14dcdc37046df2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5fe6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.624763:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000a2a6970f/0x7d14dcdc37046df2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5fe6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.624783:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.624785:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.624787:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.624822:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:2.0:1788876628.624980:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.625802:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.625809:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000cd3fa9f4/0x7d14dcdc37046a2c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a55d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1788876628.625822:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618902fc00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.625827:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000cd3fa9f4/0x7d14dcdc37046a2c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a55d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1788876628.625860:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000cd3fa9f4/0x7d14dcdc37046a2c lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a55d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1788876628.625868:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.625883:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000cd3fa9f4/0x7d14dcdc37046a2c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a55d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1788876628.626212:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000cd3fa9f4/0x7d14dcdc37046a2c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a55d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1788876628.626230:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.626232:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.626238:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.626252:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511224064:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.626443:0:15182:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876628.626456:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532270208 00000100:00100000:1.0:1788876628.626459:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532270208 00000100:00100000:1.0:1788876628.626486:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 153 00000100:00100000:1.0:1788876628.626494:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1fa8050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8455:x1875771532270208:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.626504:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.627012:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.627021:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1fa8050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8455:x1875771532270208:12345-192.168.203.111@tcp:104: Request processed in 526us (658us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.627030:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 153 00010000:00010000:2.0:1788876628.627370:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.627380:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.627411:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886cf600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.627417:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.627450:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.627458:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.627472:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.627484:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e1e09a45/0x7d14dcdc37046dc8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1da:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5f76 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.627499:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.627501:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.627502:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.627516:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788876628.627525:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511224192:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.628325:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.630479:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.640676:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511223808:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.640687:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511223680:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.641282:0:33373:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876628.641301:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532272000 00000100:00100000:1.0:1788876628.641308:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532272000 00000100:00100000:1.0:1788876628.641365:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 160 00000100:00100000:1.0:1788876628.641375:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1fa8450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43954:x1875771532272000:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.641415:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d008204f/0x7d14dcdc37046e70 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6136 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.641492:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.641499:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1fa8450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43954:x1875771532272000:12345-192.168.203.111@tcp:104: Request processed in 126us (988us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.641507:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 160 00010000:00010000:3.0:1788876628.643245:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.643258:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d008204f/0x7d14dcdc37046e70 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6136 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.643275:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876ec00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.643283:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d008204f/0x7d14dcdc37046e70 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6136 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.643339:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d008204f/0x7d14dcdc37046e70 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6136 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.643349:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.643373:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d008204f/0x7d14dcdc37046e70 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6136 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.643408:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000d008204f/0x7d14dcdc37046e70 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6136 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.643429:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.643431:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.643434:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.643456:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511225088:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.646617:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511224192:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.646822:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.646853:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.646876:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532272640 00000100:00100000:1.0:1788876628.646886:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532272640 00000100:00100000:1.0:1788876628.646938:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532272896 00000100:00100000:1.0:1788876628.646941:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532272896 00000100:00100000:1.0:1788876628.646958:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 161 00000100:00100000:1.0:1788876628.646980:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188699f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43952:x1875771532272640:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.646999:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.647121:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.647128:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188699f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43952:x1875771532272640:12345-192.168.203.111@tcp:104: Request processed in 162us (309us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.647136:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 161 00000100:00100000:1.0:1788876628.647165:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 162 00000100:00100000:1.0:1788876628.647172:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188699c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532272896:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.647182:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.648067:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876628.648077:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.648089:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876f400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.648096:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.648145:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1788876628.649196:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511224064:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.652038:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.652084:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.652096:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000086f0d80d/0x7d14dcdc37046eee lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6286 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1788876628.652125:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00010000:00010000:2.0:1788876628.652126:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.652131:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.652139:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876f000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.652146:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1788876628.652150:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511225216:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.652207:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.652213:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.652219:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.652230:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b15817e6/0x7d14dcdc37046e31 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1dc:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a608e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1788876628.652238:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511225344:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876628.652243:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.652245:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.652247:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.652398:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:1.0:1788876628.652426:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188699c00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532272896:12345-192.168.203.111@tcp:104: Request processed in 5251us (5569us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.652438:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 162 00000100:00100000:1.0:1788876628.652517:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:2.0:1788876628.652803:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.658470:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.658485:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d950ec1b/0x7d14dcdc37046e07 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a601e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.658499:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886cfe00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.658507:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d950ec1b/0x7d14dcdc37046e07 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a601e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.658691:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d950ec1b/0x7d14dcdc37046e07 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a601e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.658701:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.658726:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d950ec1b/0x7d14dcdc37046e07 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a601e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.658735:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d950ec1b/0x7d14dcdc37046e07 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1db:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a601e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.658752:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.658754:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.658756:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.660230:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.660254:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532273408 00000100:00100000:1.0:1788876628.660262:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532273408 00000100:00100000:1.0:1788876628.660309:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 164 00000100:00100000:1.0:1788876628.660318:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43961:x1875771532273408:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.660334:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.660418:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.660426:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43961:x1875771532273408:12345-192.168.203.111@tcp:104: Request processed in 108us (201us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.660445:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 164 00010000:00010000:3.0:1788876628.660448:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.660454:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.660463:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876fa00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.660468:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.660483:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.660500:0:15181:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532273920 00000100:00100000:1.0:1788876628.660503:0:15181:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532273920 00010000:00010000:3.0:1788876628.660505:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.660513:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1788876628.660522:0:15181:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 165 00010000:00010000:3.0:1788876628.660527:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.660528:0:15181:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43958:x1875771532273920:12345-192.168.203.111@tcp:104: 00010000:00010000:3.0:1788876628.660535:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e96ae2e1/0x7d14dcdc37046e9a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a61a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.660537:0:15181:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.660565:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.660566:0:15181:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00010000:00010000:3.0:1788876628.660569:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.660571:0:15181:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:LOV_OSC_UUID+4:43958:x1875771532273920:12345-192.168.203.111@tcp:104: Request processed in 44us (87us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.660577:0:15181:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 165 00010000:00010000:3.0:1788876628.660577:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876e400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.660581:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.660600:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.660606:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:3.0:1788876628.660608:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.660614:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.660618:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532274048 00000100:00100000:1.0:1788876628.660621:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532274048 00010000:00010000:3.0:1788876628.660622:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000025ff1b51/0x7d14dcdc37046ec4 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6216 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.660634:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.660636:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.660638:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 166 00010000:00010000:3.0:1788876628.660638:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.660644:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43954:x1875771532274048:12345-192.168.203.111@tcp:104: 00000100:00100000:3.0:1788876628.660650:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511225600:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.660652:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000000551572/0x7d14dcdc37046f18 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a62f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1788876628.660693:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876628.660699:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511225728:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.661181:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.661189:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43954:x1875771532274048:12345-192.168.203.111@tcp:104: Request processed in 544us (582us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.661196:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 166 00010000:00010000:1.0:1788876628.661212:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.661223:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000000551572/0x7d14dcdc37046f18 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a62f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.661231:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188690000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.661234:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000000551572/0x7d14dcdc37046f18 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a62f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.661256:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000000551572/0x7d14dcdc37046f18 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a62f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.661263:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.661278:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000000551572/0x7d14dcdc37046f18 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a62f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.661285:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000000551572/0x7d14dcdc37046f18 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a62f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.661298:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.661300:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.661302:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.662215:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.662241:0:33373:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876628.662249:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532274176 00000100:00100000:1.0:1788876628.662254:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532274176 00000100:00100000:1.0:1788876628.662284:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 167 00000100:00100000:1.0:1788876628.662291:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1fabc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43955:x1875771532274176:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.662303:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.662353:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.662359:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1fabc50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43955:x1875771532274176:12345-192.168.203.111@tcp:104: Request processed in 68us (145us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.662365:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 167 00000100:00100000:1.0:1788876628.662519:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532274688 00000100:00100000:1.0:1788876628.662523:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532274688 00000100:00100000:1.0:1788876628.662535:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532274816 00000100:00100000:1.0:1788876628.662538:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532274816 00000100:00100000:1.0:1788876628.662546:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532274944 00000100:00100000:1.0:1788876628.662549:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532274944 00000100:00100000:1.0:1788876628.662557:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532275072 00000100:00100000:1.0:1788876628.662559:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532275072 00000100:00100000:1.0:1788876628.662570:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 168 00000100:00100000:1.0:1788876628.662576:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43963:x1875771532274688:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.662585:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009073c61f/0x7d14dcdc37046a56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5646 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.662614:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876628.662619:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afd880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43963:x1875771532274688:12345-192.168.203.111@tcp:104: Request processed in 44us (162us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.662625:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 168 00000100:00100000:1.0:1788876628.662634:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532275328 00000100:00100000:1.0:1788876628.662637:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532275328 00000100:00100000:1.0:1788876628.662646:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532275456 00000100:00100000:1.0:1788876628.662649:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532275456 00000100:00100000:1.0:1788876628.662658:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 169 00000100:00100000:1.0:1788876628.662663:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43960:x1875771532274816:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.663237:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.663282:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876628.663289:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afea00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43960:x1875771532274816:12345-192.168.203.111@tcp:104: Request processed in 625us (800us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.663296:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 169 00000100:00100000:1.0:1788876628.663314:0:33373:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876628.663325:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532275712 00000100:00100000:1.0:1788876628.663328:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532275712 00000100:00100000:1.0:1788876628.663345:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 170 00000100:00100000:1.0:1788876628.663351:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afdf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43966:x1875771532274944:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.663360:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000937fab39/0x7d14dcdc37046e85 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a616e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.663470:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/4 locks to priority blp list, 4 blwis in pool 00000100:00100000:1.0:1788876628.663477:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afdf80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+6:43966:x1875771532274944:12345-192.168.203.111@tcp:104: Request processed in 125us (962us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.663483:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 170 00000100:00100000:1.0:1788876628.663496:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 171 00000100:00100000:1.0:1788876628.663502:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:17045:x1875771532275072:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.663510:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000080e102a8/0x7d14dcdc37046f42 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6366 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.663534:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/5 locks to priority blp list, 5 blwis in pool 00000100:00100000:1.0:1788876628.663539:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afed80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:17045:x1875771532275072:12345-192.168.203.111@tcp:104: Request processed in 38us (1004us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.663551:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 171 00000100:00100000:1.0:1788876628.663564:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 172 00000100:00100000:1.0:1788876628.663569:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8453:x1875771532275328:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.663578:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.663600:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/6 locks to priority blp list, 6 blwis in pool 00000100:00100000:1.0:1788876628.663605:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afc380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:8453:x1875771532275328:12345-192.168.203.111@tcp:104: Request processed in 36us (1035us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.663610:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 172 00000100:00100000:1.0:1788876628.663622:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 173 00000100:00100000:1.0:1788876628.663628:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afdc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43962:x1875771532275456:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.663636:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.663657:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/7 locks to priority blp list, 7 blwis in pool 00000100:00100000:1.0:1788876628.663662:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afdc00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43962:x1875771532275456:12345-192.168.203.111@tcp:104: Request processed in 35us (1071us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.664230:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 173 00000100:00100000:1.0:1788876628.664245:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 174 00000100:00100000:1.0:1788876628.664252:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1faa050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43964:x1875771532275712:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.664262:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.664290:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/8 locks to priority blp list, 8 blwis in pool 00000100:00100000:1.0:1788876628.664302:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1faa050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43964:x1875771532275712:12345-192.168.203.111@tcp:104: Request processed in 51us (1483us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.664309:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 174 00010000:00010000:1.0:1788876628.664349:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 7 total in blp. (7 blwis in pool) 00010000:00010000:1.0:1788876628.664357:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.664365:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061883ad000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.664370:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.664505:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.664514:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.664532:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.664540:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000009fdae704/0x7d14dcdc37046e5b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1dd:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a60fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.664562:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:1.0:1788876628.664566:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009073c61f/0x7d14dcdc37046a56 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a5646 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.664572:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061890a0200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.664575:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009073c61f/0x7d14dcdc37046a56 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a5646 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.664603:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009073c61f/0x7d14dcdc37046a56 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5646 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.664609:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.664615:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009073c61f/0x7d14dcdc37046a56 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5646 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.664622:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009073c61f/0x7d14dcdc37046a56 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a5646 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.664636:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:1.0:1788876628.664639:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.664645:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188691600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.664648:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:2.0:1788876628.665278:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511225088:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.665303:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511225344:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.665312:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511225216:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.665343:0:15184:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:2.0:1788876628.665355:0:15184:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532275840 00000100:00100000:2.0:1788876628.665360:0:15184:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532275840 00010000:00010000:1.0:1788876628.665479:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665490:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.665509:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665517:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000029e9cb2f/0x7d14dcdc37046f96 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6446 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665541:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:1.0:1788876628.665545:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000937fab39/0x7d14dcdc37046e85 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a616e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665552:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876e200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.665556:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000937fab39/0x7d14dcdc37046e85 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a616e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665588:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000937fab39/0x7d14dcdc37046e85 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a616e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665595:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.665601:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000937fab39/0x7d14dcdc37046e85 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a616e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665609:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000937fab39/0x7d14dcdc37046e85 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1de:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a616e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665624:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:1.0:1788876628.665628:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000080e102a8/0x7d14dcdc37046f42 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6366 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665634:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061883aca00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.665638:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000080e102a8/0x7d14dcdc37046f42 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6366 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665734:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000080e102a8/0x7d14dcdc37046f42 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6366 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665742:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.665748:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000080e102a8/0x7d14dcdc37046f42 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6366 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665755:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000080e102a8/0x7d14dcdc37046f42 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6366 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665768:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.665772:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.665779:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187baa800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.665783:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.665801:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.665809:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.665814:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.665820:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000002a5681bf/0x7d14dcdc37046a41 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1c4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a560e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876628.665841:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.665845:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665851:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188691000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.665855:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665876:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665882:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.665888:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665895:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000bb0a4f53/0x7d14dcdc37046f6c lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a63d6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665907:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.665911:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665917:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061883ade00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.665921:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665951:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665958:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.665962:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665979:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000007438a753/0x7d14dcdc37046eaf lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1df:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a61de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.665992:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.665993:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.665995:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.666007:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532276352 00000100:00100000:1.0:1788876628.666010:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532276352 00000100:00100000:1.0:1788876628.666031:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532276480 00000100:00100000:1.0:1788876628.666033:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532276480 00000100:00100000:1.0:1788876628.666043:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532276608 00000100:00100000:1.0:1788876628.666045:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532276608 00000100:00100000:1.0:1788876628.666059:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 175 00000100:00100000:1.0:1788876628.666065:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afe300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:8455:x1875771532276352:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.666076:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000963566f3/0x7d14dcdc37046f03 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a62be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.666110:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.666116:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afe300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:8455:x1875771532276352:12345-192.168.203.111@tcp:104: Request processed in 51us (1348us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.666129:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 175 00000100:00100000:1.0:1788876628.666144:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 176 00000100:00100000:2.0:1788876628.666824:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 27 00000100:00100000:2.0:1788876628.666834:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618838b050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:43956:x1875771532275840:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876628.666844:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008d0e0c0b/0x7d14dcdc37046ed9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a624e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.666877:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1788876628.666882:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618838b050 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:43956:x1875771532275840:12345-192.168.203.111@tcp:104: Request processed in 52us (4013us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876628.666888:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 27 00010000:00010000:2.0:1788876628.666923:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876628.666928:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000963566f3/0x7d14dcdc37046f03 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a62be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.666936:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876f200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.666941:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000963566f3/0x7d14dcdc37046f03 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a62be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667231:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000963566f3/0x7d14dcdc37046f03 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a62be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667239:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.667253:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000963566f3/0x7d14dcdc37046f03 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a62be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667260:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000963566f3/0x7d14dcdc37046f03 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a62be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667276:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.667279:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008d0e0c0b/0x7d14dcdc37046ed9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a624e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667285:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618876e000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.667288:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008d0e0c0b/0x7d14dcdc37046ed9 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a624e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667308:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008d0e0c0b/0x7d14dcdc37046ed9 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a624e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667319:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.667325:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008d0e0c0b/0x7d14dcdc37046ed9 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a624e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667332:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000008d0e0c0b/0x7d14dcdc37046ed9 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a624e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.667345:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.667347:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.667349:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.667374:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227136:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.667555:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788876628.667563:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227008:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.667640:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afe680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8456:x1875771532276480:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.667651:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000839f74b3/0x7d14dcdc37046f2d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a632e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.667931:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.667938:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afe680 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:8456:x1875771532276480:12345-192.168.203.111@tcp:104: Request processed in 1787us (3045us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.667945:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 176 00000100:00100000:1.0:1788876628.667959:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532276864 00000100:00100000:1.0:1788876628.667972:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532276864 00000100:00100000:1.0:1788876628.667992:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 177 00000100:00100000:1.0:1788876628.667999:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198afd180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532276608:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.668008:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004fa31215/0x7d14dcdc37046fc0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a64b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.668029:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876628.668034:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198afd180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532276608:12345-192.168.203.111@tcp:104: Request processed in 36us (3093us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.668040:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 177 00000100:00100000:1.0:1788876628.668050:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 178 00000100:00100000:1.0:1788876628.668055:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06198aff480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43952:x1875771532276864:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.668062:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.668107:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876628.668112:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06198aff480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43952:x1875771532276864:12345-192.168.203.111@tcp:104: Request processed in 58us (1115us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.668118:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 178 00010000:00010000:1.0:1788876628.668135:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.668140:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000839f74b3/0x7d14dcdc37046f2d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a632e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.668147:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061883ac200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.668231:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876628.668235:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004fa31215/0x7d14dcdc37046fc0 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a64b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668242:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188691a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.668245:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004fa31215/0x7d14dcdc37046fc0 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a64b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668273:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004fa31215/0x7d14dcdc37046fc0 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a64b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668280:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.668292:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004fa31215/0x7d14dcdc37046fc0 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a64b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668299:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004fa31215/0x7d14dcdc37046fc0 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a64b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668314:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.668317:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668324:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061883ac800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.668327:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668349:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668356:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.668361:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668368:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000054aa0ec6/0x7d14dcdc37046f57 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a639e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.668458:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.668462:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.668464:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.668477:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227392:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.668513:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788876628.668520:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227264:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.668578:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000839f74b3/0x7d14dcdc37046f2d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a632e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.668604:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000839f74b3/0x7d14dcdc37046f2d lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a632e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.668612:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.668643:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000839f74b3/0x7d14dcdc37046f2d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a632e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.668653:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000839f74b3/0x7d14dcdc37046f2d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a632e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.668845:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.668847:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.668849:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.668857:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.669522:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511225728:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.669536:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511225600:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.679884:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511226240:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.679897:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511226112:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.679907:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511225984:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.680528:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.680554:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532277504 00000100:00100000:1.0:1788876628.680560:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532277504 00000100:00100000:1.0:1788876628.680594:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 180 00000100:00100000:1.0:1788876628.680603:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188699f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43953:x1875771532277504:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.680616:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005c9a7a0f/0x7d14dcdc37047014 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6596 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.682971:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876628.682980:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188699f80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+5:43953:x1875771532277504:12345-192.168.203.111@tcp:104: Request processed in 2377us (2453us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.682987:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 180 00010000:00010000:1.0:1788876628.683009:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788876628.683016:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004efb374d/0x7d14dcdc37046fea lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6526 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.683024:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188691400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.683030:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004efb374d/0x7d14dcdc37046fea lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6526 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.683078:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004efb374d/0x7d14dcdc37046fea lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6526 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.683096:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.683117:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004efb374d/0x7d14dcdc37046fea lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6526 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.683127:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000004efb374d/0x7d14dcdc37046fea lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6526 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.683144:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788876628.683148:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005c9a7a0f/0x7d14dcdc37047014 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6596 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.683472:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.684265:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227264:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.684277:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227392:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.684286:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227008:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.684294:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511227136:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788876628.684488:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886cf800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.684494:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005c9a7a0f/0x7d14dcdc37047014 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6596 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.684527:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005c9a7a0f/0x7d14dcdc37047014 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6596 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.684535:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.684545:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005c9a7a0f/0x7d14dcdc37047014 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6596 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.684554:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005c9a7a0f/0x7d14dcdc37047014 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6596 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.684570:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.684579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.684581:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876628.684602:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511227520:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.684612:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511226368:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.684620:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511226624:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.684627:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511226880:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.684635:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511226496:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.684643:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511226752:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.687796:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.687826:0:33373:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532278016 00000100:00100000:1.0:1788876628.687833:0:33373:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532278016 00000100:00100000:1.0:1788876628.687888:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 182 00000100:00100000:1.0:1788876628.687897:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532278016:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.687917:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.688001:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.688008:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:8455:x1875771532278016:12345-192.168.203.111@tcp:104: Request processed in 112us (220us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.688016:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 182 00010000:00010000:3.0:1788876628.689812:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.689826:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.689845:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06190249200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.689853:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.689906:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.689914:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.689944:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.689953:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000f01a6d9b/0x7d14dcdc37047068 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6676 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.689983:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.689985:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.689987:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.690247:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228032:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.697744:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:2.0:1788876628.699245:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.699254:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000081609886/0x7d14dcdc37047092 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a66e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.699264:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06190249c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.699270:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000081609886/0x7d14dcdc37047092 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a66e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.699317:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000081609886/0x7d14dcdc37047092 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a66e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.699325:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.699347:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000081609886/0x7d14dcdc37047092 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a66e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.699356:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000081609886/0x7d14dcdc37047092 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a66e6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.699378:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.699492:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.699496:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.699511:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511228160:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.704256:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.704292:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.704980:0:33373:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 185 00000100:00100000:1.0:1788876628.704997:0:33373:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43961:x1875771532279424:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.705007:0:33373:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009d1b7ad/0x7d14dcdc37047053 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a663e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.705039:0:33373:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:2.0:1788876628.705043:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228032:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.705044:0:33373:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_002:LOV_OSC_UUID+4:43961:x1875771532279424:12345-192.168.203.111@tcp:104: Request processed in 54us (753us total) trans 0 rc 0/0 00010000:00010000:3.0:1788876628.708580:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1788876628.708596:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000047679f27/0x7d14dcdc37046fd5 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a64ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708612:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188690800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.708621:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000047679f27/0x7d14dcdc37046fd5 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a64ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708681:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000047679f27/0x7d14dcdc37046fd5 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a64ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708690:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.708717:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000047679f27/0x7d14dcdc37046fd5 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a64ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708726:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000047679f27/0x7d14dcdc37046fd5 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a64ee expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708747:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.708752:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009d1b7ad/0x7d14dcdc37047053 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a663e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708759:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06190249000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.708764:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009d1b7ad/0x7d14dcdc37047053 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a663e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708784:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009d1b7ad/0x7d14dcdc37047053 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a663e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.708791:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.708797:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009d1b7ad/0x7d14dcdc37047053 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a663e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.710299:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000009d1b7ad/0x7d14dcdc37047053 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a663e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.710330:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.710334:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.710336:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.710359:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228288:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.710474:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876628.710484:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228416:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876628.711899:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:3.0:1788876628.711907:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.711917:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06188690600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.711923:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.711981:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.711989:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.712007:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712016:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f0d72b4c/0x7d14dcdc37046fab lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a647e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712035:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1788876628.712039:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c8a282c7/0x7d14dcdc37046a6b lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a567e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1788876628.712053:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06181493c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.712057:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c8a282c7/0x7d14dcdc37046a6b lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a567e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1788876628.712078:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c8a282c7/0x7d14dcdc37046a6b lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a567e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1788876628.712085:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.712091:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c8a282c7/0x7d14dcdc37046a6b lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a567e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1788876628.712098:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c8a282c7/0x7d14dcdc37046a6b lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1c5:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a567e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 1 00010000:00010000:3.0:1788876628.712110:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.712114:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ec2f1794/0x7d14dcdc37047029 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a65ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712121:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06190248000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.712125:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ec2f1794/0x7d14dcdc37047029 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a65ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712140:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ec2f1794/0x7d14dcdc37047029 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a65ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712153:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876628.712156:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ec2f1794/0x7d14dcdc37047029 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a65ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712163:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000ec2f1794/0x7d14dcdc37047029 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e8:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a65ce expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712177:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.712178:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876628.712180:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.712194:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228544:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.712234:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228800:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.712255:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876628.712261:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228672:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876628.712290:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876628.712754:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511228160:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1788876628.712777:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788876628.712782:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876628.712791:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886ec200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:3.0:1788876628.712795:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876628.714471:0:33373:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 185 00010000:00010000:1.0:1788876628.714513:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876628.714520:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000083fc000a/0x7d14dcdc3704707d lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a66ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.714529:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06190249e00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876628.714534:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000083fc000a/0x7d14dcdc3704707d lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a66ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.714576:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000083fc000a/0x7d14dcdc3704707d lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a66ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.714583:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876628.714613:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000083fc000a/0x7d14dcdc3704707d lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a66ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.714620:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 0000000083fc000a/0x7d14dcdc3704707d lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1ea:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a66ae expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.714639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.714646:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876628.714648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.729026:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.729046:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.729077:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.729087:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000005f64fce9/0x7d14dcdc37046fff lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e7:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a655e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.729112:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.729114:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.729117:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.729450:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511229056:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.729856:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876628.729887:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532280192 00000100:00100000:1.0:1788876628.729894:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532280192 00000100:00100000:1.0:1788876628.729939:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 191 00000100:00100000:1.0:1788876628.729948:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532280192:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876628.729974:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c4b8ca1b/0x7d14dcdc37046f81 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a640e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876628.730513:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876628.730523:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869bb80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43958:x1875771532280192:12345-192.168.203.111@tcp:104: Request processed in 574us (667us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876628.730530:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 191 00010000:00010000:2.0:1788876628.733026:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876628.733039:0:33372:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c4b8ca1b/0x7d14dcdc37046f81 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a640e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.733053:0:33372:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06187bab200 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876628.733062:0:33372:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c4b8ca1b/0x7d14dcdc37046f81 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a640e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.733121:0:33372:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c4b8ca1b/0x7d14dcdc37046f81 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a640e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.733130:0:33372:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876628.733152:0:33372:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c4b8ca1b/0x7d14dcdc37046f81 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a640e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.733161:0:33372:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000c4b8ca1b/0x7d14dcdc37046f81 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1e4:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a640e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876628.733180:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.733182:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876628.733184:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876628.733198:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788876628.733216:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511229184:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876628.734881:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511228928:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.737503:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228672:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.737520:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228544:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.737530:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228800:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.737540:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228416:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876628.737550:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511228288:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.753859:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511229184:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:3.0:1788876628.753874:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511229056:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876629.763595:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876630.783340:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876631.169645:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511229312:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876631.169678:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511229568:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876631.169690:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511229824:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876631.169700:0:2395:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477396 gen 1 00000100:00100000:1.0:1788876631.169958:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511229440:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876631.169974:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511229696:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876631.805717:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876632.830806:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876633.854295:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876634.569765:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.569813:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.569849:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.569879:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.569905:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.569954:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.569978:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.569992:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.570005:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:2.0:1788876634.570020:0:15184:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532285312 00000100:00100000:1.0:1788876634.570027:0:2388:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:2.0:1788876634.570036:0:15184:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532285312 00000100:00100000:1.0:1788876634.570062:0:15182:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876634.570082:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532284800 00000100:00100000:1.0:1788876634.570087:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532284800 00000100:00100000:2.0:1788876634.570093:0:15184:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532285440 00000100:00100000:2.0:1788876634.570098:0:15184:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532285440 00000100:00100000:1.0:1788876634.570118:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532284928 00000100:00100000:2.0:1788876634.570120:0:15184:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532285568 00000100:00100000:1.0:1788876634.570123:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532284928 00000100:00100000:2.0:1788876634.570126:0:15184:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532285568 00000100:00100000:1.0:1788876634.570142:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532285056 00000100:00100000:1.0:1788876634.570146:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532285056 00000100:00100000:1.0:1788876634.570156:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532285184 00000100:00100000:1.0:1788876634.570159:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532285184 00000100:00100000:1.0:1788876634.570172:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 192 00000100:00100000:1.0:1788876634.570182:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:43952:x1875771532284800:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876634.570204:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008b3a02ae/0x7d14dcdc37047110 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6836 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.570289:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876634.570301:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869b100 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:43952:x1875771532284800:12345-192.168.203.111@tcp:104: Request processed in 117us (547us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876634.570308:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 192 00000100:00100000:1.0:1788876634.570326:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 193 00000100:00100000:1.0:1788876634.570332:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:43958:x1875771532284928:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876634.570351:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000809338e5/0x7d14dcdc3704718e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6986 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.570403:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to priority blp list, 2 blwis in pool 00000100:00100000:1.0:1788876634.570409:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869ad80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:43958:x1875771532284928:12345-192.168.203.111@tcp:104: Request processed in 76us (597us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876634.570415:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 193 00000100:00100000:1.0:1788876634.570427:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 194 00000100:00100000:1.0:1788876634.570434:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:43966:x1875771532285056:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876634.570442:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.570470:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/3 locks to priority blp list, 3 blwis in pool 00000100:00100000:1.0:1788876634.570475:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698000 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+6:43966:x1875771532285056:12345-192.168.203.111@tcp:104: Request processed in 43us (627us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876634.570481:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 194 00000100:00100000:1.0:1788876634.570492:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 195 00000100:00100000:1.0:1788876634.570498:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06186250c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+5:8455:x1875771532285184:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876634.572669:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e066b60f/0x7d14dcdc370471a3 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a69be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:2.0:1788876634.572824:0:15184:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532285696 00000100:00100000:2.0:1788876634.572831:0:15184:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532285696 00000100:00100000:2.0:1788876634.572844:0:15184:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532285824 00000100:00100000:2.0:1788876634.572848:0:15184:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532285824 00000100:00100000:2.0:1788876634.572857:0:15184:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532286080 00000100:00100000:2.0:1788876634.572860:0:15184:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532286080 00000100:00100000:2.0:1788876634.572873:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 28 00000100:00100000:2.0:1788876634.572883:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:10319:x1875771532285312:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876634.572888:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/10 locks to priority blp list, 10 blwis in pool 00010000:00010000:2.0:1788876634.572895:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007a419005/0x7d14dcdc37046a80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a56b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000100:00100000:1.0:1788876634.572898:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06186250c50 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+7:8455:x1875771532285184:12345-192.168.203.111@tcp:104: Request processed in 2398us (3017us total) trans 0 rc 0/0 00010000:00010000:2.0:1788876634.572951:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/11 locks to priority blp list, 11 blwis in pool 00000100:00100000:2.0:1788876634.572957:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869a300 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:10319:x1875771532285312:12345-192.168.203.111@tcp:104: Request processed in 77us (3051us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876634.572963:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 28 00000100:00100000:2.0:1788876634.572979:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 29 00000100:00100000:2.0:1788876634.572986:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:17045:x1875771532285440:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876634.572998:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.573021:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/12 locks to priority blp list, 12 blwis in pool 00000100:00100000:2.0:1788876634.573028:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698e00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:17045:x1875771532285440:12345-192.168.203.111@tcp:104: Request processed in 41us (3074us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876634.573033:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 29 00000100:00100000:2.0:1788876634.573046:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 30 00000100:00100000:2.0:1788876634.573052:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa0618869b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:43964:x1875771532285568:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876634.573060:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000466c7b4/0x7d14dcdc37047125 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a686e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.573084:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/13 locks to priority blp list, 13 blwis in pool 00000100:00100000:2.0:1788876634.573089:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa0618869b480 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+7:43964:x1875771532285568:12345-192.168.203.111@tcp:104: Request processed in 38us (3112us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876634.573095:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 30 00000100:00100000:2.0:1788876634.573105:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 31 00000100:00100000:2.0:1788876634.573110:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+6:43954:x1875771532285696:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876634.573118:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:1.0:1788876634.573125:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 195 00010000:00010000:2.0:1788876634.573161:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/14 locks to priority blp list, 14 blwis in pool 00000100:00100000:2.0:1788876634.573168:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698a80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:43954:x1875771532285696:12345-192.168.203.111@tcp:104: Request processed in 57us (3175us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876634.573174:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 31 00000100:00100000:2.0:1788876634.573191:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 32 00010000:00010000:1.0:1788876634.573193:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 13 total in blp. (13 blwis in pool) 00010000:00010000:1.0:1788876634.573214:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008b3a02ae/0x7d14dcdc37047110 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6836 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573223:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886b3a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876634.573229:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008b3a02ae/0x7d14dcdc37047110 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6836 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573278:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008b3a02ae/0x7d14dcdc37047110 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6836 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573286:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876634.573309:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008b3a02ae/0x7d14dcdc37047110 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6836 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573317:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000008b3a02ae/0x7d14dcdc37047110 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6836 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573342:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 12 total in blp. (12 blwis in pool) 00010000:00010000:1.0:1788876634.573346:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000809338e5/0x7d14dcdc3704718e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6986 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573352:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618902e000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876634.573356:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000809338e5/0x7d14dcdc3704718e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6986 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573373:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000809338e5/0x7d14dcdc3704718e lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6986 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573411:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876634.573419:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000809338e5/0x7d14dcdc3704718e lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6986 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573426:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000809338e5/0x7d14dcdc3704718e lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6986 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573441:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 11 total in blp. (11 blwis in pool) 00010000:00010000:1.0:1788876634.573444:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573451:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886b3c00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876634.573455:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573477:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573484:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876634.573489:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.573496:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000db52dfbb/0x7d14dcdc370470fb lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a67fe expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:2.0:1788876634.574291:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188699880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:43953:x1875771532285824:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876634.574303:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005b40149b/0x7d14dcdc370470bc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6756 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574340:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/12 locks to priority blp list, 12 blwis in pool 00000100:00100000:2.0:1788876634.574346:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188699880 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+5:43953:x1875771532285824:12345-192.168.203.111@tcp:104: Request processed in 64us (4339us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876634.574352:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 32 00000100:00100000:2.0:1788876634.574375:0:15184:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 33 00000100:00100000:2.0:1788876634.574398:0:15184:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa06188698700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:43961:x1875771532286080:12345-192.168.203.111@tcp:104: 00010000:00010000:2.0:1788876634.574408:0:15184:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000bc6b3fb9/0x7d14dcdc370470a7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a671e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574509:0:15184:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/13 locks to priority blp list, 13 blwis in pool 00000100:00100000:2.0:1788876634.574515:0:15184:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa06188698700 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb01_001:LOV_OSC_UUID+4:43961:x1875771532286080:12345-192.168.203.111@tcp:104: Request processed in 134us (4488us total) trans 0 rc 0/0 00000100:00100000:2.0:1788876634.574522:0:15184:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 33 00010000:00010000:2.0:1788876634.574551:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 12 total in blp. (12 blwis in pool) 00010000:00010000:2.0:1788876634.574558:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b8e250d5/0x7d14dcdc3704714f lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a68de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574566:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886edc00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.574572:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b8e250d5/0x7d14dcdc3704714f lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a68de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574602:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b8e250d5/0x7d14dcdc3704714f lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a68de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574611:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.574638:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b8e250d5/0x7d14dcdc3704714f lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a68de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574648:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000b8e250d5/0x7d14dcdc3704714f lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a68de expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574667:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 11 total in blp. (11 blwis in pool) 00010000:00010000:2.0:1788876634.574671:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009dc2e470/0x7d14dcdc370470e6 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a67c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574678:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06190248400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.574681:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009dc2e470/0x7d14dcdc370470e6 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a67c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574699:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009dc2e470/0x7d14dcdc370470e6 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a67c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574707:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.574713:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009dc2e470/0x7d14dcdc370470e6 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a67c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574720:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000009dc2e470/0x7d14dcdc370470e6 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1ed:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a67c6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574736:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 10 total in blp. (10 blwis in pool) 00010000:00010000:2.0:1788876634.574740:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000019ef4d93/0x7d14dcdc37047164 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6916 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574746:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886ed600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.574750:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000019ef4d93/0x7d14dcdc37047164 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6916 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574767:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000019ef4d93/0x7d14dcdc37047164 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6916 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574775:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.574779:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000019ef4d93/0x7d14dcdc37047164 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6916 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574786:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 0000000019ef4d93/0x7d14dcdc37047164 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6916 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574799:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 9 total in blp. (9 blwis in pool) 00010000:00010000:2.0:1788876634.574803:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f1ac4034/0x7d14dcdc370470d1 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a678e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574810:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886ed400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.574814:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f1ac4034/0x7d14dcdc370470d1 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a678e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574830:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f1ac4034/0x7d14dcdc370470d1 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a678e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574842:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.574846:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f1ac4034/0x7d14dcdc370470d1 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a678e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574855:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000f1ac4034/0x7d14dcdc370470d1 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a678e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574868:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 8 total in blp. (8 blwis in pool) 00010000:00010000:2.0:1788876634.574872:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001d6c7552/0x7d14dcdc37047179 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a694e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574879:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618902f000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.574883:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001d6c7552/0x7d14dcdc37047179 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a694e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574900:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001d6c7552/0x7d14dcdc37047179 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a694e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574907:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.574911:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001d6c7552/0x7d14dcdc37047179 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a694e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574919:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000001d6c7552/0x7d14dcdc37047179 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1f0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a694e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574945:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 7 total in blp. (7 blwis in pool) 00010000:00010000:2.0:1788876634.574949:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ac1d6708/0x7d14dcdc3704713a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a68a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574957:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886b3400 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.574961:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ac1d6708/0x7d14dcdc3704713a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a68a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574979:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ac1d6708/0x7d14dcdc3704713a lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a68a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574986:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.574991:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ac1d6708/0x7d14dcdc3704713a lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a68a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.574999:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000ac1d6708/0x7d14dcdc3704713a lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1ef:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a68a6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575013:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 6 total in blp. (6 blwis in pool) 00010000:00010000:2.0:1788876634.575017:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e066b60f/0x7d14dcdc370471a3 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a69be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575023:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886b2000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.575027:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e066b60f/0x7d14dcdc370471a3 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a69be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575046:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e066b60f/0x7d14dcdc370471a3 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a69be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575058:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.575063:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e066b60f/0x7d14dcdc370471a3 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a69be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575070:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000e066b60f/0x7d14dcdc370471a3 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1f1:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a69be expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575083:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 5 total in blp. (5 blwis in pool) 00010000:00010000:2.0:1788876634.575087:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007a419005/0x7d14dcdc37046a80 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a56b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876634.575093:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06181492000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.575097:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007a419005/0x7d14dcdc37046a80 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a56b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876634.575113:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007a419005/0x7d14dcdc37046a80 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a56b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876634.575120:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.575124:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007a419005/0x7d14dcdc37046a80 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a56b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876634.575130:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000007a419005/0x7d14dcdc37046a80 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1c6:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a56b6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876634.575157:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 4 total in blp. (4 blwis in pool) 00010000:00010000:2.0:1788876634.575161:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575168:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886b3000 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.575172:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.575189:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576501:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.576518:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576529:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000e9c47304/0x7d14dcdc370471b8 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a69f6 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576553:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 3 total in blp. (3 blwis in pool) 00010000:00010000:2.0:1788876634.576568:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000466c7b4/0x7d14dcdc37047125 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a686e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576576:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886b2800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.576582:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000466c7b4/0x7d14dcdc37047125 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a686e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576627:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000466c7b4/0x7d14dcdc37047125 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a686e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576639:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.576645:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000466c7b4/0x7d14dcdc37047125 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a686e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576652:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000466c7b4/0x7d14dcdc37047125 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1ee:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a686e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576668:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 2 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1788876634.576672:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576678:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06180030800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.576682:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576701:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576708:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.576713:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.576720:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 000000000897132f/0x7d14dcdc370471cd lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1f2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6a2e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.577736:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876634.577764:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000bc6b3fb9/0x7d14dcdc370470a7 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a671e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.577777:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06190248a00 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876634.577785:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000bc6b3fb9/0x7d14dcdc370470a7 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a671e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.577828:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000bc6b3fb9/0x7d14dcdc370470a7 lrc: 2/0,0 mode: --/PW res: [0x280000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a671e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.577836:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876634.577845:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000bc6b3fb9/0x7d14dcdc370470a7 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a671e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.577852:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000bc6b3fb9/0x7d14dcdc370470a7 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x1eb:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a671e expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.577870:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876634.577872:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876634.577875:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876634.577911:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511229952:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876634.578018:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511230208:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876634.578053:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788876634.578061:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511230080:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876634.578080:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511231744:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876634.578174:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876634.578184:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005b40149b/0x7d14dcdc370470bc lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6756 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.578266:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886ec800 already unused, calling callback (ffffffffc106c300) 00010000:00010000:2.0:1788876634.578272:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005b40149b/0x7d14dcdc370470bc lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6756 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.578306:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005b40149b/0x7d14dcdc370470bc lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6756 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.578314:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876634.578320:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005b40149b/0x7d14dcdc370470bc lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6756 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.578329:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 000000005b40149b/0x7d14dcdc370470bc lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1ec:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6756 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876634.578354:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876634.578358:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876634.578361:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876634.578373:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876634.578426:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230336:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578469:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230592:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578499:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230848:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578537:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231104:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578569:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231360:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578603:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231616:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578635:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 6 async RPCs [0->1] 00000100:00100000:2.0:1788876634.578643:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230464:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578667:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230720:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.578694:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230976:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.579269:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231232:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.581361:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231488:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.581549:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231872:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876634.601763:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00000100:00100000:1.0:1788876634.601830:0:15182:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/2 left), rc = 0 00000100:00100000:1.0:1788876634.601859:0:15182:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1875771532286976 00000100:00100000:1.0:1788876634.601870:0:15182:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1875771532286976 00000100:00100000:1.0:1788876634.601949:0:15182:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.111@tcp, seq: 202 00000100:00100000:1.0:1788876634.601961:0:15182:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffffa061a1fa9450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532286976:12345-192.168.203.111@tcp:104: 00010000:00010000:1.0:1788876634.601982:0:15182:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.602067:0:15182:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1788876634.602074:0:15182:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffffa061a1fa9450 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:LOV_OSC_UUID+4:43959:x1875771532286976:12345-192.168.203.111@tcp:104: Request processed in 116us (326us total) trans 0 rc 0/0 00000100:00100000:1.0:1788876634.602082:0:15182:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.111@tcp, seq: 202 00010000:00010000:1.0:1788876634.602135:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788876634.602143:0:15192:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x420000010000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.602152:0:15192:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886b2600 already unused, calling callback (ffffffffc106c300) 00010000:00010000:1.0:1788876634.602159:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x428400010000 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.602210:0:15192:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 2/0,0 mode: --/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.602218:0:15192:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876634.602244:0:15192:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 1/0,0 mode: --/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.602252:0:15192:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000b15ffe52/0x7d14dcdc370471e2 lrc: 0/0,0 mode: --/PW res: [0x2c0000401:0x1f3:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x4c29400010020 nid: local remote: 0xf5bf0f7e313a6a66 expref: -99 pid: 32098 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876634.602287:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876634.602289:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876634.602291:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876634.602301:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788876634.602321:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511232000:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.605983:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230336:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606014:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230592:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606026:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230848:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606037:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231104:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606047:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231360:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606058:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231616:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606108:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230464:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606118:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230720:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.606127:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511230976:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.607833:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231232:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.607863:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231488:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788876634.607874:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511231872:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876634.610030:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511232000:192.168.203.111@tcp:103:ldlm_bl.0 00000080:00200000:1.0:1788876635.539494:0:33948:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876635.539534:0:33948:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876635.539606:0:33948:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33948 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876635.539618:0:33948:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33948 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876635.539629:0:33948:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33948 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876635.539660:0:33948:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33948:1875771511232128:192.168.203.111@tcp:101:lfs.0 00000100:00100000:1.0:1788876635.539772:0:33948:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063300 going to sleep for 11 seconds 00000100:00100000:3.0:1788876635.547079:0:33948:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33948:1875771511232128:192.168.203.111@tcp:101:lfs.0 00010000:00010000:3.0:1788876635.547112:0:33948:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547127:0:33948:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547150:0:33948:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547163:0:33948:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547178:0:33948:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547187:0:33948:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547195:0:33948:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876635.547213:0:33948:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869a680 x1875771511232128/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876651 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876635.547230:0:33948:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547245:0:33948:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876635.547252:0:33948:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876635.547289:0:33948:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876635.547296:0:33948:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048832 00000080:00010000:3.0:1788876635.547302:0:33948:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876635.547306:0:33948:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048832 from it ffffb95ac1d1fb58 00010000:00010000:3.0:1788876635.547309:0:33948:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547319:0:33948:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fa00/0x7d14dcdc37048832 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae969 expref: -99 pid: 33948 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876635.547348:0:33948:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876635.547425:0:33948:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=13 00000080:00200000:3.0:1788876635.547431:0:33948:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876635.547459:0:33948:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d6a.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876635.547502:0:33948:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887bc00/0x7d14dcdc37048839 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33948 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547510:0:33948:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887bc00/0x7d14dcdc37048839 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33948 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876635.547524:0:33948:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887bc00/0x7d14dcdc37048839 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33948 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876635.547545:0:33948:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33948:1875771511232256:192.168.203.111@tcp:101:lfs.0 00000100:00100000:3.0:1788876635.547639:0:33948:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864dbcc0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876635.572051:0:33948:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33948:1875771511232512:192.168.203.111@tcp:36:lfs.0 00000080:00200000:3.0:1788876635.572113:0:33948:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x107:0x0](ffffa06198283248) 00000080:00200000:3.0:1788876635.572122:0:33948:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x107:0x0], updating i_size 4096 00000080:00200008:3.0:1788876635.572175:0:33948:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 300000 00010000:00010000:2.0:1788876635.902089:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876636.285899:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511232768:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.286029:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511233024:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.286068:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1788876636.286074:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511232640:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.286108:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511232896:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.286584:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511233152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.302309:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511232640:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.302340:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511232896:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.302354:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511233152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.302370:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511232768:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876636.302420:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511233024:192.168.203.111@tcp:400:kworker.0 00000080:00200008:2.0:1788876636.616941:0:33961:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876636.616960:0:33961:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876636.616990:0:33961:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876636.617066:0:33961:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33961 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.617077:0:33961:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33961 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.617843:0:33961:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33961 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876636.617884:0:33961:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233280:192.168.203.111@tcp:101:lfs.0 00000100:00100000:2.0:1788876636.618050:0:33961:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168240 going to sleep for 11 seconds 00000100:00100000:2.0:1788876636.629073:0:33961:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477397 gen 1 00000100:00100000:2.0:1788876636.629125:0:33961:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233280:192.168.203.111@tcp:101:lfs.0 00010000:00010000:2.0:1788876636.629152:0:33961:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629166:0:33961:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629179:0:33961:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629191:0:33961:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629206:0:33961:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629217:0:33961:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629225:0:33961:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876636.629238:0:33961:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888af100 x1875771511233280/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876652 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876636.629256:0:33961:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629269:0:33961:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876636.629277:0:33961:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876636.629315:0:33961:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876636.629323:0:33961:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048840 00000080:00010000:2.0:1788876636.629329:0:33961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876636.629333:0:33961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048840 from it ffffb95ac163bcb8 00010000:00010000:2.0:1788876636.629337:0:33961:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876636.629344:0:33961:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876636.630183:0:33961:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876636.630192:0:33961:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876636.630213:0:33961:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876636.630228:0:33961:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876636.630241:0:33961:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876636.630249:0:33961:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876636.630256:0:33961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876636.630260:0:33961:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048840 00000080:00010000:2.0:1788876636.630265:0:33961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048840 from it ffffb95ac163bcb8 00010000:00010000:2.0:1788876636.630269:0:33961:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876636.630277:0:33961:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876636.630472:0:33961:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876636.630484:0:33961:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876636.630491:0:33961:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876636.630499:0:33961:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876636.630506:0:33961:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876636.630510:0:33961:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876636.630514:0:33961:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048840 00000080:00010000:2.0:1788876636.630517:0:33961:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048840 from it ffffb95ac163bbb0 00010000:00010000:2.0:1788876636.630521:0:33961:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876636.630528:0:33961:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876636.630538:0:33961:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876636.630546:0:33961:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876636.630648:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:2.0:1788876636.630665:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00800000:00000004:2.0:1788876636.630673:0:33961:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1604, max_age 1627 00000100:00100000:2.0:1788876636.630719:0:33961:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233408:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876636.630813:0:33961:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168240 going to sleep for 11 seconds 00000100:00100000:2.0:1788876636.637244:0:33961:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233408:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876636.637282:0:33961:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023707/1024000 00000080:00200080:2.0:1788876636.637375:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:2.0:1788876636.637440:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00800000:00000004:2.0:1788876636.637450:0:33961:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1604, max_age 1627 00000100:00100000:2.0:1788876636.637492:0:33961:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233536:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788876636.637850:0:33961:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168240 going to sleep for 11 seconds 00000100:00100000:3.0:1788876636.648540:0:33961:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233536:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876636.648578:0:33961:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:3.0:1788876636.648677:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.648699:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.648708:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.648712:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.648716:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.648719:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.648723:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.648726:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.648730:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.648733:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.648737:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.648740:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.648744:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.648748:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.648752:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649033:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649045:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649052:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649056:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649060:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649064:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649068:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649072:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649076:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649080:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649083:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649099:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649103:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649108:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649112:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649116:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649120:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649124:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649128:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649132:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649136:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649140:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649144:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649148:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649151:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649155:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649159:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649163:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649166:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649170:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649173:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649177:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649181:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649185:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649188:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649192:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649195:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649199:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649203:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649207:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649210:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649213:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649217:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649220:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649224:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649227:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649231:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649235:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649239:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649242:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649245:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649249:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649252:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649256:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649259:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649263:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649266:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649270:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649273:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649276:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649279:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649283:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649286:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649291:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649294:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649298:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649302:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649306:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649309:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649313:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649316:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649319:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649324:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649327:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649331:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649334:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649338:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649341:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649344:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649348:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649351:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649355:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649358:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649362:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649365:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649369:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649374:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649378:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649413:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649418:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649423:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649427:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649430:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649433:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649437:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649440:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649443:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649447:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649450:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649453:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649455:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649459:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649462:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649465:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649467:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649471:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649473:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649477:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649480:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649484:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00800000:00000080:3.0:1788876636.649486:0:33961:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.649490:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.649497:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00020000:00000004:3.0:1788876636.649507:0:33961:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1604, max_age 1627 00000100:00100000:3.0:1788876636.649557:0:33961:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233664:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876636.649869:0:33961:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864db840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876636.655348:0:33961:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233664:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876636.655418:0:33961:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788876636.655480:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.655498:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00020000:00000004:3.0:1788876636.655505:0:33961:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1604, max_age 1627 00000100:00100000:3.0:1788876636.655541:0:33961:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233792:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876636.655622:0:33961:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864db840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876636.659515:0:33961:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:33961:1875771511233792:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876636.659545:0:33961:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788876636.659612:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659628:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659635:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659640:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659645:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659649:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659653:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659657:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659662:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659666:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659677:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659682:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659687:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659691:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659696:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659700:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659704:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659708:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659713:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659717:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659723:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659727:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659731:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659734:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659739:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659743:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659748:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659752:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659756:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659760:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659765:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659768:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659773:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659777:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659781:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659785:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659791:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659795:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659799:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659803:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659808:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659817:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659822:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659825:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659829:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659833:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659836:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659840:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.659843:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.659846:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660065:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660072:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660077:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660080:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660083:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660086:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660090:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660093:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660097:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660101:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660105:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660108:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660112:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660115:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660119:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660123:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660126:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660129:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660134:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660137:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660141:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660144:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660148:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660152:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660156:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660160:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660164:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660167:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660171:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660174:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660178:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660182:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660197:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660202:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660207:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660210:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660214:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660218:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660221:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660224:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660228:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660232:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660236:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660239:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660243:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660247:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660251:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660255:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660259:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660263:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660267:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660270:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660274:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660277:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660281:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660284:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660287:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660290:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660294:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660298:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660302:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660305:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660309:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660312:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660316:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660320:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660323:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660327:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660331:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660334:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660339:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660343:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660347:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660350:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200080:3.0:1788876636.660354:0:33961:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffcea678e20 00020000:00000080:3.0:1788876636.660358:0:33961:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388400 uarg=0000000000000000 00000080:00200008:3.0:1788876636.660373:0:33961:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:1.0:1788876636.730323:0:33968:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876636.730351:0:33968:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876636.730372:0:33968:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876636.730424:0:33968:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876636.730444:0:33968:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876636.730452:0:33968:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876636.730456:0:33968:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048840 00000080:00010000:1.0:1788876636.730463:0:33968:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048840 from it ffffb95ac1c2fbd0 00010000:00010000:1.0:1788876636.730467:0:33968:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876636.730477:0:33968:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1788876636.730493:0:33968:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876636.730506:0:33968:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:1.0:1788876636.730510:0:33968:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876636.730516:0:33968:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876636.730523:0:33968:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876636.730531:0:33968:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1788876636.730537:0:33968:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876636.730540:0:33968:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876636.730543:0:33968:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048840 00000080:00010000:1.0:1788876636.730546:0:33968:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048840 from it ffffb95ac1c2fcd0 00010000:00010000:1.0:1788876636.730549:0:33968:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876636.730564:0:33968:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:1.0:1788876636.730572:0:33968:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876636.730577:0:33968:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876636.730582:0:33968:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876636.730588:0:33968:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876636.730596:0:33968:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:1.0:1788876636.730601:0:33968:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876636.730604:0:33968:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876636.730607:0:33968:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048840 00000080:00010000:1.0:1788876636.730610:0:33968:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048840 from it ffffb95ac1c2fdb0 00010000:00010000:1.0:1788876636.730612:0:33968:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1788876636.730619:0:33968:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:1.0:1788876636.730625:0:33968:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876636.730630:0:33968:0:(namei.c:2062:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:1.0:1788876636.730634:0:33968:0:(namei.c:2017:ll_mknod()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:1.0:1788876636.730656:0:33968:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876636.730714:0:33968:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:1.0:1788876636.730721:0:33968:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876636.730724:0:33968:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048840 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae9a1 expref: -99 pid: 33961 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000020:00100000:1.0:1788876636.730748:0:33968:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876636.730759:0:33968:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188699500 x1875771511234048/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876636.730781:0:33968:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:33968:1875771511234048:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:1.0:1788876636.730852:0:33968:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063300 going to sleep for 11 seconds 00000100:00100000:1.0:1788876636.735343:0:33968:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:33968:1875771511234048:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:1.0:1788876636.735530:0:33968:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x108:0x0](ffffa061981c6c08) 00000080:00200000:1.0:1788876636.735571:0:33968:0:(namei.c:2070:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, unhashed 1 00000080:00200000:1.0:1788876636.765200:0:33971:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876636.765235:0:33971:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876636.765305:0:33971:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 33971 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.765317:0:33971:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33971 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.765329:0:33971:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 33971 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876636.765354:0:33971:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33971:1875771511234176:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876636.765481:0:33971:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063300 going to sleep for 11 seconds 00000100:00100000:1.0:1788876636.780948:0:33971:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33971:1875771511234176:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876636.780986:0:33971:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781000:0:33971:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781013:0:33971:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781026:0:33971:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781040:0:33971:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781049:0:33971:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781057:0:33971:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876636.781071:0:33971:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188699180 x1875771511234176/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876652 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876636.781088:0:33971:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781126:0:33971:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876636.781135:0:33971:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876636.781172:0:33971:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876636.781180:0:33971:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048847 00000080:00010000:1.0:1788876636.781187:0:33971:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876636.781190:0:33971:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048847 from it ffffb95ac1c2fb58 00010000:00010000:1.0:1788876636.781194:0:33971:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781203:0:33971:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876636.781232:0:33971:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876636.781259:0:33971:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:1.0:1788876636.781264:0:33971:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:1.0:1788876636.781297:0:33971:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: fsa-oleg311-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788876636.781344:0:33971:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 33971 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781352:0:33971:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33971 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.781362:0:33971:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 33971 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876636.781437:0:33971:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33971:1875771511234304:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876636.781536:0:33971:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063300 going to sleep for 11 seconds 00000100:00100000:1.0:1788876636.790071:0:33971:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33971:1875771511234304:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876636.790120:0:33971:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790148:0:33971:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790157:0:33971:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790170:0:33971:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790183:0:33971:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790197:0:33971:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790205:0:33971:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790213:0:33971:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876636.790228:0:33971:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188699180 x1875771511234304/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 600/584 e 0 to 0 dl 1788876652 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876636.790243:0:33971:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790273:0:33971:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/264 (0 0) 00000080:00200000:1.0:1788876636.790280:0:33971:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876636.790308:0:33971:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x108:0x0], updating i_size 0 00000080:00200000:1.0:1788876636.790312:0:33971:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x108:0x0](ffffa061981c6c08): rc = 0 00000080:00200000:1.0:1788876636.790331:0:33971:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x108:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1788876636.790338:0:33971:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x108:0x0](ffffa061981c6c08) for lock 0x7d14dcdc3704884e 00000080:00010000:1.0:1788876636.790350:0:33971:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x108:0x0](ffffa061981c6c08) 00000080:00010000:1.0:1788876636.790354:0:33971:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704884e from it ffffb95ac1c2fb20 00010000:00010000:1.0:1788876636.790358:0:33971:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876636.790367:0:33971:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1788876636.790452:0:33971:0:(file.c:6296:ll_getattr_dentry()) START file fsa-oleg311-client.virtnet:[0x200001b71:0x108:0x0](ffffa061981c6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876636.790460:0:33971:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x108:0x0](ffffa061981c6c08),name=fsa-oleg311-client.virtnet 00000002:00010000:1.0:1788876636.790486:0:33971:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x108:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876636.790496:0:33971:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876636.790507:0:33971:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876636.790514:0:33971:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876636.790518:0:33971:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x108:0x0](ffffa061981c6c08) 00000080:00010000:1.0:1788876636.790522:0:33971:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x108:0x0](ffffa061981c6c08) for lock 0x7d14dcdc3704884e 00000080:00010000:1.0:1788876636.790533:0:33971:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704884e from it ffffb95ac1c2fcb8 00010000:00010000:1.0:1788876636.790536:0:33971:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876636.790544:0:33971:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876636.790617:0:33971:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876636.790626:0:33971:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876636.790645:0:33971:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876636.790654:0:33971:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876636.790661:0:33971:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876636.790666:0:33971:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876636.790668:0:33971:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048847 00000080:00010000:1.0:1788876636.790672:0:33971:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048847 from it ffffb95ac1c2fc10 00010000:00010000:1.0:1788876636.790675:0:33971:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876636.790682:0:33971:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876636.790690:0:33971:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876636.790700:0:33971:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg311-client.virtnet, flags=0 00000080:00200000:1.0:1788876636.790706:0:33971:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876636.790719:0:33971:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876636.790726:0:33971:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876636.790735:0:33971:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876636.790741:0:33971:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876636.790746:0:33971:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876636.790748:0:33971:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048847 00000080:00010000:1.0:1788876636.790752:0:33971:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048847 from it ffffb95ac1c2fd68 00010000:00010000:1.0:1788876636.790755:0:33971:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876636.790763:0:33971:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876636.790771:0:33971:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876636.790777:0:33971:0:(namei.c:2351:ll_unlink()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876636.790798:0:33971:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876636.790822:0:33971:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876636.790831:0:33971:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x108:0x0]: losing layout lock 00010000:00010000:1.0:1788876636.790870:0:33971:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876636.790888:0:33971:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876636.790894:0:33971:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876636.790896:0:33971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc3704884e lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x108:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313ae9d9 expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876636.790929:0:33971:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ee00/0x7d14dcdc37048847 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313ae9bd expref: -99 pid: 33971 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876636.790952:0:33971:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876636.790962:0:33971:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188699180 x1875771511234560/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876636.790991:0:33971:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33971:1875771511234560:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876636.791081:0:33971:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063300 going to sleep for 11 seconds 00000100:00100000:1.0:1788876636.800101:0:33971:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:33971:1875771511234560:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876636.800180:0:33971:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x108:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788876636.800192:0:33971:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x108:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876636.800201:0:33971:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x108:0x0](ffffa061981c6c08) 00010000:00010000:2.0:1788876638.974196:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876641.405690:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511234816:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.405790:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511235072:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.405825:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1788876641.405832:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511234688:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.405862:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511234944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.405897:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511235200:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.410635:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511235200:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.410651:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511235072:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.412115:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511234944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.412132:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511234816:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876641.413218:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511234688:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876642.046829:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876643.299570:0:34037:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:2.0:1788876643.317439:0:34038:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876643.317468:0:34038:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876643.317532:0:34038:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 34038 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.317545:0:34038:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34038 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.317556:0:34038:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34038 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876643.317581:0:34038:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:34038:1875771511235328:192.168.203.111@tcp:101:mcreate.0 00000100:00100000:2.0:1788876643.317666:0:34038:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788876643.326126:0:34038:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:34038:1875771511235328:192.168.203.111@tcp:101:mcreate.0 00010000:00010000:2.0:1788876643.326155:0:34038:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326168:0:34038:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326181:0:34038:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326192:0:34038:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326204:0:34038:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326213:0:34038:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326220:0:34038:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876643.326236:0:34038:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888ae680 x1875771511235328/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876659 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876643.326251:0:34038:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326265:0:34038:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876643.326271:0:34038:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876643.326296:0:34038:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876643.326302:0:34038:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048855 00000080:00010000:2.0:1788876643.326307:0:34038:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876643.326310:0:34038:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048855 from it ffffb95ac1d07bd0 00010000:00010000:2.0:1788876643.326313:0:34038:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326322:0:34038:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876643.326349:0:34038:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876643.326367:0:34038:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=24 00000080:00200000:2.0:1788876643.326372:0:34038:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=0 00000002:00010000:2.0:1788876643.326412:0:34038:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d6a.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:2.0:1788876643.326453:0:34038:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 34038 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326461:0:34038:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34038 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.326475:0:34038:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34038 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876643.326496:0:34038:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:34038:1875771511235456:192.168.203.111@tcp:101:mcreate.0 00000100:00100000:2.0:1788876643.326565:0:34038:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788876643.334109:0:34038:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:34038:1875771511235456:192.168.203.111@tcp:101:mcreate.0 00010000:00010000:2.0:1788876643.334136:0:34038:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334158:0:34038:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334167:0:34038:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334179:0:34038:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334190:0:34038:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334203:0:34038:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334211:0:34038:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334225:0:34038:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876643.334238:0:34038:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffffa061888ae680 x1875771511235456/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/688 e 0 to 0 dl 1788876659 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876643.334253:0:34038:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334264:0:34038:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/263 (0 0) 00000080:00200000:2.0:1788876643.334270:0:34038:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788876643.334300:0:34038:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x107:0x0], updating i_size 4096 00000080:00200000:2.0:1788876643.334304:0:34038:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x107:0x0](ffffa06198283248): rc = 0 00000080:00010000:2.0:1788876643.334308:0:34038:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x107:0x0](ffffa06198283248) for lock 0x7d14dcdc3704885c 00000080:00010000:2.0:1788876643.334321:0:34038:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x107:0x0](ffffa06198283248) 00000080:00010000:2.0:1788876643.334325:0:34038:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704885c from it ffffb95ac1d07aa0 00010000:00010000:2.0:1788876643.334328:0:34038:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876643.334337:0:34038:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876643.334593:0:34038:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x107:0x0](ffffa06198283248), inode mode 41ed mask 1 00000080:00200000:2.0:1788876643.334618:0:34038:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffffa06198283248), flags=1536 00000080:00200000:2.0:1788876643.334623:0:34038:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x107:0x0](ffffa06198283248), inode mode 41ed mask 3 00000080:00200000:2.0:1788876643.334630:0:34038:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x107:0x0](ffffa06198283248), inode mode 41ed mask 3 00000080:00200000:2.0:1788876643.334635:0:34038:0:(namei.c:2062:ll_create_nd()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffffa06198283248), flags=33188, excl=1 00000080:00200000:2.0:1788876643.334639:0:34038:0:(namei.c:2017:ll_mknod()) VFS Op:name=f6a.replay-single, dir=[0x200001b71:0x107:0x0](ffffa06198283248) mode 100644 dev 0 00000020:00100000:2.0:1788876643.334704:0:34038:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876643.334711:0:34038:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ae680 x1875771511235712/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876643.334732:0:34038:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:34038:1875771511235712:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876643.334800:0:34038:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788876643.339561:0:34038:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:34038:1875771511235712:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:3.0:1788876643.339671:0:34038:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x109:0x0](ffffa06198286c08) 00000080:00200000:3.0:1788876643.339712:0:34038:0:(namei.c:2070:ll_create_nd()) VFS Op:name=f6a.replay-single, unhashed 1 00010000:00010000:2.0:1788876645.118529:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876646.526570:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511235968:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876646.526669:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236224:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876646.526703:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1788876646.526710:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511235840:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876646.526740:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876646.526762:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876646.535373:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876646.535420:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236352:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876646.535431:0:2397:0:(recover.c:205:ptlrpc_request_handle_notconn()) import lustre-MDT0000-mdc-ffffa06189393000 of lustre-MDT0000_UUID@192.168.203.111@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1788876646.535437:0:2397:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1788876646.535441:0:2397:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa06189393000: Connection to lustre-MDT0000 (at 192.168.203.111@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1788876646.561962:0:2397:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876646.561978:0:2397:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1587 00000100:00080000:2.0:1788876646.561994:0:2397:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876646.562017:0:2397:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876646.562053:0:2397:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511236480/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876646.562110:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511236480:192.168.203.111@tcp:38:ptlrpcd_01_00.0 00000100:00100000:2.0:1788876646.571280:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511235968:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876646.571325:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236224:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876646.572179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876646.572187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876646.572192:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511236480:192.168.203.111@tcp:38:ptlrpcd_01_00.0 00010000:00010000:2.0:1788876648.189561:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876651.261658:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788876651.645736:0:16024:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876651.645747:0:16024:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788876651.645752:0:16024:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876651.645766:0:16024:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1638 00000100:00080000:2.0:1788876651.645779:0:16024:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1788876651.645788:0:16024:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876651.645810:0:16024:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876651.645831:0:16024:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511236736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876651.646320:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511236608:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876651.646465:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236992:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876651.646502:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1788876651.646510:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236864:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876651.646536:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511237120:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876651.653672:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876651.653694:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876651.653700:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511236736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876651.654867:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236864:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876651.654888:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511237120:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876651.654900:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511236992:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876654.333680:0:17748:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1788876654.333766:0:17748:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/6 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1788876654.337790:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511237248:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876654.337917:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788876654.337926:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511237376:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1788876654.343726:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876654.348295:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511237376:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1788876654.348337:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511237248:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00080000:2.0:1788876656.765671:0:16024:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876656.765679:0:16024:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788876656.765683:0:16024:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876656.765695:0:16024:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1643 00000100:00080000:2.0:1788876656.765709:0:16024:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1788876656.765718:0:16024:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876656.765731:0:16024:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876656.765751:0:16024:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af800 x1875771511237632/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876656.766658:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511237760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876656.766716:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511238016:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876656.766748:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788876656.766756:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511237504:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876656.766798:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511237888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876656.773879:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511237888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876656.773897:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511237760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876656.773908:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511238016:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876656.773989:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876656.773996:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876656.774000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511237632:192.168.203.111@tcp:38:kworker.0 00010000:00010000:2.0:1788876657.407995:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876660.477597:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876661.888656:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876661.888663:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876661.888667:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876661.888678:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1648 00000100:00080000:3.0:1788876661.888690:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1788876661.888697:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876661.888709:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876661.888727:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511238272/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876661.888813:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511238400:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876661.888915:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511238144:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876661.902890:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511238528:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876661.909259:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511238656:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876661.909744:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876661.909752:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876661.909757:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511238272:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876661.909946:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511238528:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876661.912097:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511238656:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876661.913722:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511238400:192.168.203.111@tcp:400:kworker.0 00000100:00000400:3.0:1788876662.917738:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876646/real 1788876646] req@ffffa061888ac700 x1875771511235840/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876662 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876662.917772:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888ac700 x1875771511235840/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876662 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876662.917783:0:2397:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1788876662.917787:0:2397:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.111@tcp: Connection to MGS (at 192.168.203.111@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1788876662.917792:0:2397:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.111@tcp for MGC192.168.203.111@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1788876662.917795:0:2397:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1788876662.917799:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888ac700 x1875771511235840/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876662 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876662.917809:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888ace00 x1875771511236608/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876667 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876662.917819:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888ac380 x1875771511237504/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876672 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876662.917833:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188872a00 x1875771511238144/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876677 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1788876662.917841:0:2397:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1788876662.917844:0:2397:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1788876662.917845:0:2397:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1788876662.917847:0:2397:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:3.0:1788876662.917877:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511238144:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.917886:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511236608:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.917890:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511235840:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.917928:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511237504:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876662.917943:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876662.917948:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788876662.917951:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876662.917957:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1587 00000100:00080000:3.0:1788876662.917968:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788876662.917998:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511238784/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788876662.918017:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876662.918020:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876662.918022:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876662.918026:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1653 00000100:00080000:3.0:1788876662.918029:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788876662.918032:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876662.918042:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876662.918053:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511238912/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876662.918099:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511238784:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788876662.918185:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511238912:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876662.918222:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511239040:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.918248:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511239296:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.918270:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876662.918277:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511239168:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.929177:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511239168:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.929201:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511239296:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876662.935859:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511239040:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876662.935908:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876662.935914:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876662.935918:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511238912:192.168.203.111@tcp:38:kworker.0 00010000:00010000:2.0:1788876663.549814:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876666.623910:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876668.031156:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511239424:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788876668.047637:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876668.047645:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876668.047649:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511239424:192.168.203.111@tcp:38:kworker.0 00010000:00010000:2.0:1788876669.694039:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876672.128348:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876662/real 1788876662] req@ffffa06188872a00 x1875771511238784/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876672 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876672.128408:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511238784/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876672 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876672.128430:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876672.128459:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788876672.128463:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511238784:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788876672.129264:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511239936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788876672.129333:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511240192:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876672.129368:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511240064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788876672.129483:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511240448:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876672.140099:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876672.140120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-11) 00000100:00100000:3.0:1788876672.140126:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511240064:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788876672.151071:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) MGC192.168.203.111@tcp: connect to target with instance 0 10000000:01000000:3.0:1788876672.151097:0:2394:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1788876672.151119:0:2394:0:(import.c:1328:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.203.111@tcp) after server handle changed from 0xf5bf0f7e313a8620 to 0xf5bf0f7e313aeaea 00000100:00080000:3.0:1788876672.151127:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1788876672.151134:0:2394:0:(import.c:1718:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.203.111@tcp; invalidating 00000100:00080000:3.0:1788876672.151643:0:34261:0:(import.c:1642:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.203.111@tcp to MGS@192.168.203.111@tcp 00000100:00100000:3.0:1788876672.151648:0:34261:0:(import.c:323:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1788876672.151652:0:34261:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1788876672.151682:0:34261:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061889ad400/0x7d14dcdc37047bdd lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313a86f2 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788876672.151695:0:34261:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061889ad400/0x7d14dcdc37047bdd lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313a86f2 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876672.151702:0:34261:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1788876672.151715:0:34261:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788876672.151726:0:34261:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061889ad400/0x7d14dcdc37047bdd lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313a86f2 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876672.151745:0:34261:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06188934800/0x7d14dcdc37047beb lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313a8700 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788876672.151751:0:34261:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06188934800/0x7d14dcdc37047beb lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313a8700 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876672.151757:0:34261:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1788876672.151761:0:34261:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788876672.151764:0:34261:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06188934800/0x7d14dcdc37047beb lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313a8700 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876672.151781:0:34261:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06188978600/0x7d14dcdc37047bd6 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313a86eb expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788876672.151787:0:34261:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06188978600/0x7d14dcdc37047bd6 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313a86eb expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876672.151793:0:34261:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1788876672.151797:0:34261:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788876672.151801:0:34261:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06188978600/0x7d14dcdc37047bd6 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313a86eb expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876672.151816:0:34261:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06188979800/0x7d14dcdc37047be4 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313a86f9 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788876672.151823:0:34261:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06188979800/0x7d14dcdc37047be4 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313a86f9 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876672.151828:0:34261:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1788876672.151831:0:34261:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788876672.151835:0:34261:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06188979800/0x7d14dcdc37047be4 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313a86f9 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:3.0:1788876672.151850:0:34261:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:3.0:1788876672.151855:0:34261:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from RECOVER to FULL 10000000:01000000:3.0:1788876672.151857:0:34261:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1788876672.151861:0:34261:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:3.0:1788876672.151864:0:34261:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) MGC192.168.203.111@tcp: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00010000:00010000:2.0:1788876672.766790:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1788876673.534617:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1788876673.534624:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffffa06189393000 from 68 10000000:01000000:3.0:1788876673.534630:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1788876673.534704:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.534715:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.534730:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876673.534755:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511240576:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.534851:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.545072:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511240576:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876673.545105:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb76 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.545119:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313aeb76 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876673.545190:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511240704:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.545282:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.558265:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511240704:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00080000:3.0:1788876673.558329:0:15194:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06183c27000 already not connected (conn 112, was 0): DISCONN 00010000:00080000:3.0:1788876673.558344:0:15194:0:(ldlm_lib.c:85:import_set_conn()) imp ffffa06183c27000@lustre-MDT0000-mdc-ffffa06189393000: found existing conn 192.168.203.111@tcp, moved to head 00000100:00080000:3.0:1788876673.558351:0:15194:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876673.558369:0:15194:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876673.558399:0:15194:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876673.560779:0:15194:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511240832/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 10000000:01000000:3.0:1788876673.560849:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1788876673.560862:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb76 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.560873:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb76 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876673.560883:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1788876673.560885:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06189393000 from 52 10000000:01000000:3.0:1788876673.560889:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1788876673.560919:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.560928:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.560939:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876673.560959:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511240960:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.561057:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.561079:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511240832:192.168.203.111@tcp:38:ll_cfg_requeue.0 00010000:00080000:3.0:1788876673.567548:0:2394:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa06188873b80 x1875771511240832/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/416 e 0 to 0 dl 1788876698 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876673.567574:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa06189393000: connect to target with instance 17 00000100:00080000:3.0:1788876673.567593:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1788876673.567601:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876673.567604:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477397 gen 1 00000100:00100000:3.0:1788876673.567610:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188699500 x1875771511234048/t68719477398(68719477398) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876652 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876673.567619:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 68719477397 last 0 00000100:00080000:3.0:1788876673.567625:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa06188699500 x1875771511234048/t68719477398(68719477398) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876652 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876673.567758:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511240832:192.168.203.111@tcp:38:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.567790:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511234048:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:3.0:1788876673.576950:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511240960:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876673.576984:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb7d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.576997:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313aeb7d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876673.577078:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241088:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.577193:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.583839:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241088:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.583914:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241216:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.584001:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.587105:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241216:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1788876673.588024:0:34307:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 51 00000020:01000000:3.0:1788876673.588193:0:15194:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 52-51 (rc=0) 10000000:01000000:3.0:1788876673.588211:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1788876673.588225:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb7d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.588237:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb7d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876673.588249:0:15194:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1788876673.588251:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06189393000 from 7 10000000:01000000:3.0:1788876673.588254:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1788876673.588295:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.588302:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.588312:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876673.588335:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241344:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.588552:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.591438:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241344:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876673.591471:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb84 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.591494:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313aeb84 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876673.591567:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241472:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.591635:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.595729:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241472:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.595805:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241600:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1788876673.597624:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab540 going to sleep for 11 seconds 00000100:00100000:2.0:1788876673.607278:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241600:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1788876673.614112:0:34308:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:1.0:1788876673.615580:0:15194:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:1.0:1788876673.615616:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:1.0:1788876673.615635:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb84 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876673.615663:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb84 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876673.615675:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:1.0:1788876673.615679:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffffa0618906b380 from 1 10000000:01000000:1.0:1788876673.615682:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:1.0:1788876673.615732:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876673.615741:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876673.615753:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876673.615788:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241728:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1788876673.615893:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c649c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.626940:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241728:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876673.626974:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb8b expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.626988:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313aeb8b expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876673.627069:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241856:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876673.627182:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eabd80 going to sleep for 11 seconds 00000100:00100000:3.0:1788876673.643917:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511241856:192.168.203.111@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1788876673.644151:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1788876673.644163:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb8b expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876673.644174:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313aeb8b expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876675.839722:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876677.250312:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level REPLAY/5 force 0 force_next 1 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876677.250318:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: REPLAY) 00000100:00100000:3.0:1788876677.250662:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511241984:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876677.255477:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242112:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876677.258679:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511242240:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876677.266729:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242368:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876677.286448:0:2397:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1788876677.286465:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242112:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876677.286484:0:2397:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 2 blwis in pool 00000100:00100000:2.0:1788876677.286488:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242368:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876677.286517:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876677.286531:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876677.286536:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876677.286538:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876677.286539:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876677.286563:0:2398:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1788876677.286570:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511242240:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876677.286586:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876677.286592:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876677.286593:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876677.286595:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876677.289763:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876677.291439:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511241984:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876677.630354:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa06188699500 x1875771511234048/t68719477398(68719477398) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876689 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.630372:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876677.630411:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477397 gen 1 00000100:00100000:3.0:1788876677.630418:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188699500 x1875771511234048/t68719477398(68719477398) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876689 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.630425:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 68719477397 last 68719477398 00000100:00080000:3.0:1788876677.630434:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa06188699180 x1875771511234560/t68719477399(68719477399) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876652 ref 1 fl New:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876677.630445:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511234048:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:3.0:1788876677.630468:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511234560:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876677.644444:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477398 gen 1 00000100:00100000:3.0:1788876677.644479:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188699180 x1875771511234560/t68719477399(68719477399) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876693 ref 2 fl Rpc:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.644494:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa06188699180 x1875771511234560/t68719477399(68719477399) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876693 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.644502:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876677.644505:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477398 gen 1 00000100:00100000:3.0:1788876677.644508:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188699180 x1875771511234560/t68719477399(68719477399) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876693 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.644514:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 68719477398 last 68719477399 00000100:00080000:3.0:1788876677.644521:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888ae680 x1875771511235712/t68719477400(68719477400) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876659 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876677.644538:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511234560:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876677.644562:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511235712:192.168.203.111@tcp:36:mcreate.0 00000100:00080000:3.0:1788876677.654804:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888ae680 x1875771511235712/t68719477400(68719477400) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876693 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.654820:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876677.654823:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477398 gen 1 00000100:00100000:3.0:1788876677.654830:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188699180 x1875771511234560/t68719477399(68719477399) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876693 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.654838:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 68719477398 last 68719477400 00000100:00080000:3.0:1788876677.654843:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00010000:00080000:3.0:1788876677.655715:0:34319:0:(ldlm_request.c:2972:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffffa06189393000 to lustre-MDT0000_UUID@192.168.203.111@tcp 00010000:00010000:3.0:1788876677.655719:0:34319:0:(ldlm_request.c:2888:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffffa06189393000 (2) 00010000:00010000:3.0:1788876677.655773:0:34319:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030a00/0x7d14dcdc3704885c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0xf5bf0f7e313aea1f expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876677.655798:0:34319:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048855 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0xf5bf0f7e313aea03 expref: -99 pid: 34038 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876677.655808:0:34319:0:(ldlm_request.c:2901:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffffa06189393000 00000100:00080000:3.0:1788876677.656200:0:34319:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:3.0:1788876677.670118:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 68719477400 gen 1 00000100:00080000:3.0:1788876677.670148:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:3.0:1788876677.670162:0:2394:0:(client.c:3248:ptlrpc_resend_req()) @@@ going to resend req@ffffa06188871500 x1875771511242496/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788876693 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.670173:0:2394:0:(client.c:3257:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa06188871500 x1875771511242496/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788876693 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876677.670181:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:1.0:1788876677.671658:0:2394:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffffa06189393000: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00000100:00100000:1.0:1788876677.671667:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511242496:192.168.203.111@tcp:400:ldlm_lock_repla.0 00010000:00010000:2.0:1788876677.885767:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876678.269627:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242752:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876678.269720:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511243008:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876678.269751:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:3.0:1788876678.269758:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511242624:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876678.269788:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242880:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876678.269810:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511243136:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876678.276880:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242880:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876678.276900:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511243136:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876678.276910:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511242752:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876678.276920:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511243008:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876678.278722:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511242624:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876678.909679:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876679.933886:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876680.972454:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1788876681.101305:0:34362:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876681.101320:0:34362:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876681.101344:0:34362:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876681.102138:0:34362:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 34362 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.102150:0:34362:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34362 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.102162:0:34362:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 34362 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876681.102190:0:34362:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243264:192.168.203.111@tcp:101:lfs.0 00000100:00100000:2.0:1788876681.102335:0:34362:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864dbcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876681.106210:0:34362:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243264:192.168.203.111@tcp:101:lfs.0 00010000:00010000:2.0:1788876681.106237:0:34362:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106251:0:34362:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106265:0:34362:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106277:0:34362:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106290:0:34362:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106300:0:34362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106308:0:34362:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876681.106319:0:34362:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888af800 x1875771511243264/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876697 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876681.106334:0:34362:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106349:0:34362:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876681.106355:0:34362:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876681.106621:0:34362:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876681.106630:0:34362:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876681.106636:0:34362:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876681.106639:0:34362:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac0d4bcb8 00010000:00010000:2.0:1788876681.106643:0:34362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876681.106654:0:34362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876681.107716:0:34362:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876681.107726:0:34362:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876681.107829:0:34362:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876681.107845:0:34362:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876681.107860:0:34362:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876681.107868:0:34362:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876681.107874:0:34362:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876681.107878:0:34362:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876681.107884:0:34362:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac0d4bcb8 00010000:00010000:2.0:1788876681.107887:0:34362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876681.107897:0:34362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876681.108019:0:34362:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876681.108029:0:34362:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876681.108037:0:34362:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876681.108114:0:34362:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876681.108123:0:34362:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876681.108128:0:34362:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876681.108131:0:34362:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876681.108135:0:34362:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac0d4bbb0 00010000:00010000:2.0:1788876681.108139:0:34362:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876681.108147:0:34362:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876681.108155:0:34362:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876681.108163:0:34362:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876681.108322:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.108337:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00800000:00000004:2.0:1788876681.108344:0:34362:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1628, max_age 1671 00000100:00100000:2.0:1788876681.108410:0:34362:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243392:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876681.108502:0:34362:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864dbcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876681.120706:0:34362:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243392:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876681.120735:0:34362:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023706/1024000 00000080:00200080:2.0:1788876681.120864:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.120882:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00800000:00000004:2.0:1788876681.120890:0:34362:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1628, max_age 1671 00000100:00100000:2.0:1788876681.120926:0:34362:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243520:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876681.121001:0:34362:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864dbcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876681.138011:0:34362:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243520:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876681.138042:0:34362:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:2.0:1788876681.138126:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138143:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138152:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138156:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138160:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138164:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138168:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138171:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138175:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138178:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138182:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138186:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138190:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138193:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138197:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138201:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138206:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138209:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138213:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138215:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138219:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138222:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138226:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138229:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138233:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138236:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138240:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138243:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138247:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138249:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138254:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138256:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138260:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138263:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138266:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138269:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138273:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138276:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138280:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138283:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138286:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138290:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138293:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138297:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138301:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138304:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138308:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138312:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138316:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138320:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138323:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138326:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138330:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138333:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138337:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138340:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138344:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138347:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138351:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138355:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138358:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138362:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138366:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138369:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138373:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.138376:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.138426:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139104:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139117:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139123:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139128:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139132:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139136:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139140:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139144:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139147:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139151:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139155:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139160:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139163:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139167:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139171:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139175:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139178:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139182:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139185:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139189:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139192:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139197:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139200:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139203:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139207:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139212:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139215:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139218:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139222:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139225:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139228:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139232:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139235:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139239:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139242:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139246:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139249:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139253:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139257:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139261:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139264:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139268:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139272:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139275:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139278:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139281:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139285:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139288:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139292:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139296:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139299:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139302:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139305:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139309:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139312:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139316:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139320:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139324:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00800000:00000080:2.0:1788876681.139327:0:34362:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.139331:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.139336:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00020000:00000004:2.0:1788876681.139345:0:34362:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1628, max_age 1671 00000100:00100000:2.0:1788876681.139407:0:34362:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243648:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876681.139613:0:34362:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864dbcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876681.148338:0:34362:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243648:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876681.148369:0:34362:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876681.148474:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.148490:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00020000:00000004:2.0:1788876681.148499:0:34362:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1628, max_age 1671 00000100:00100000:2.0:1788876681.148532:0:34362:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243776:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876681.149159:0:34362:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061864dbcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876681.152708:0:34362:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34362:1875771511243776:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876681.152738:0:34362:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788876681.152805:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152822:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152829:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152833:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152837:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152841:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152845:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152848:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152852:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152855:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152859:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152862:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152865:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152869:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152872:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152875:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152878:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152881:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152885:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152888:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152892:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152896:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152899:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152902:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152906:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152910:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152913:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152917:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152920:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152923:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152927:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152930:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152934:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152937:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152941:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152944:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152948:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152951:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152955:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152958:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152962:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152965:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152969:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152972:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152975:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152979:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152982:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152986:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152990:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.152993:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.152998:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153001:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153005:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153008:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153012:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153016:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153019:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153022:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153026:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153030:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153034:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153037:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153041:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153426:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153444:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153451:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153455:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153459:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153462:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153466:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153470:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153474:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153478:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153482:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153486:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153491:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153495:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153499:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153502:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153506:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153510:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153514:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153518:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153522:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153526:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153529:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153534:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153537:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153541:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153545:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00000080:00200080:2.0:1788876681.153548:0:34362:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe9d0662c0 00020000:00000080:2.0:1788876681.153552:0:34362:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05400 uarg=0000000000000000 00010000:00010000:3.0:1788876681.981499:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876683.006303:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876683.391030:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511243904:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.391262:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244160:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.391301:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244416:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.391338:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876683.391345:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244032:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.391374:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244288:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.400990:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244416:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.401033:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244032:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.401046:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244288:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.401056:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511243904:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876683.401067:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511244160:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876684.030694:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876685.059361:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876687.257055:0:34643:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg311-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 00010000:00010000:3.0:1788876688.125532:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876688.521137:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511244672:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876688.521156:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511244928:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876688.521166:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511244544:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876688.521176:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511244800:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876688.521188:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511245056:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876690.173609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1788876690.806006:0:34830:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876690.806021:0:34830:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876690.806043:0:34830:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876690.806062:0:34830:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876690.806078:0:34830:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876690.806086:0:34830:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876690.806093:0:34830:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876690.806096:0:34830:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876690.806103:0:34830:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac0c87cb8 00010000:00010000:2.0:1788876690.806108:0:34830:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876690.806117:0:34830:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:3.0:1788876690.813049:0:34830:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1788876690.813059:0:34830:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876690.813082:0:34830:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876690.813100:0:34830:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876690.813117:0:34830:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788876690.813124:0:34830:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876690.813132:0:34830:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876690.813137:0:34830:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:3.0:1788876690.813145:0:34830:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac0c87cb8 00010000:00010000:3.0:1788876690.813149:0:34830:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876690.813159:0:34830:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876690.813311:0:34830:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876690.813322:0:34830:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876690.813329:0:34830:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876690.813339:0:34830:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876690.813345:0:34830:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876690.813349:0:34830:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876690.813352:0:34830:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:3.0:1788876690.813356:0:34830:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac0c87bb0 00010000:00010000:3.0:1788876690.813359:0:34830:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876690.813367:0:34830:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:3.0:1788876690.813375:0:34830:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:3.0:1788876690.813416:0:34830:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:3.0:1788876690.813519:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.813545:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00800000:00000004:3.0:1788876690.813553:0:34830:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1672, max_age 1681 00000100:00100000:3.0:1788876690.813593:0:34830:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245184:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788876690.813673:0:34830:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876690.830614:0:34830:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245184:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876690.830645:0:34830:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023706/1024000 00000080:00200080:3.0:1788876690.830716:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.830733:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00800000:00000004:3.0:1788876690.830741:0:34830:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1672, max_age 1681 00000100:00100000:3.0:1788876690.830774:0:34830:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245312:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788876690.830847:0:34830:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876690.834076:0:34830:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245312:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876690.834108:0:34830:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:3.0:1788876690.834166:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834182:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834190:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834194:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834199:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834202:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834206:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834209:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834213:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834216:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834220:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834223:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834227:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834230:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834246:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834252:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834257:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834260:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834263:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834266:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834270:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834273:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834277:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834280:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834284:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834288:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834291:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834295:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834298:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834302:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834306:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834309:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834312:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834316:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834319:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834323:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834326:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834329:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834332:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834335:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834338:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834341:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834345:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834348:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834351:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834355:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834359:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834362:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834365:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834368:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834372:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834375:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834379:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834437:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834443:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834447:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834451:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834456:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834459:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834463:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834468:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834472:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834477:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834481:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834497:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834502:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834508:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834512:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834516:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834519:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834534:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834539:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834543:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834546:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834551:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834554:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834558:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834562:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834566:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834570:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834575:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834578:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834582:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834586:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834589:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834593:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834597:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834600:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834603:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834607:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834610:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834614:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834617:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834621:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834624:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834628:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834631:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834635:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834638:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834641:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834645:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834649:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834653:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834656:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834660:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834664:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834668:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834671:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834675:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834679:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834683:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834687:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834691:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834694:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834698:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834701:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834706:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834709:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834713:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834717:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834722:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834725:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834729:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834733:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834737:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00800000:00000080:3.0:1788876690.834741:0:34830:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.834745:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.834751:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00020000:00000004:3.0:1788876690.834758:0:34830:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1672, max_age 1681 00000100:00100000:3.0:1788876690.834794:0:34830:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245440:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876690.834862:0:34830:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876690.842688:0:34830:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245440:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876690.842719:0:34830:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788876690.842794:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.842813:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00020000:00000004:3.0:1788876690.842820:0:34830:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1672, max_age 1681 00000100:00100000:3.0:1788876690.842856:0:34830:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245568:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876690.843061:0:34830:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a0063840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876690.856864:0:34830:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:34830:1875771511245568:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876690.856895:0:34830:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788876690.857071:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857090:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857098:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857103:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857108:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857111:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857115:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857119:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857123:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857127:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857134:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857138:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857141:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857145:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857148:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857151:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857155:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857159:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857165:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857169:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857173:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857177:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857181:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857184:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857188:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857191:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857196:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857199:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857203:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857206:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857210:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857214:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857218:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857221:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857225:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857228:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857233:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857249:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857254:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857257:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857261:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857263:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857267:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857271:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857275:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857278:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857283:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857286:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857290:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857294:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857298:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857301:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857305:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857308:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857312:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857316:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857319:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857322:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857324:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857327:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857330:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857333:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857337:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857341:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857344:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857348:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857351:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857354:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857358:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857361:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857376:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857380:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857423:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857427:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857431:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857434:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857438:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857441:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857444:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857447:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857451:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857454:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857458:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857461:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857465:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857468:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857472:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857474:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857478:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857481:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857485:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857659:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857669:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857673:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857677:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857681:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857684:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857688:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857692:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857695:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857699:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857703:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857707:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857710:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857714:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857718:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857722:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857725:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857729:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857732:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857737:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857741:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857745:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857748:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857752:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857755:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857758:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857761:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857766:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857769:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857773:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857776:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857780:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857783:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876690.857787:0:34830:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd69e7aeb0 00020000:00000080:3.0:1788876690.857790:0:34830:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200008:3.0:1788876690.857805:0:34830:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:2.0:1788876690.898774:0:34846:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876690.898804:0:34846:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876690.898927:0:34846:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1788876690.898946:0:34846:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:2.0:1788876690.898954:0:34846:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876690.898962:0:34846:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876690.898966:0:34846:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876690.898975:0:34846:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac0dd3b58 00010000:00010000:2.0:1788876690.898979:0:34846:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1788876690.898989:0:34846:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:2.0:1788876690.899003:0:34846:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876690.899016:0:34846:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:2.0:1788876690.899020:0:34846:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d6a.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:2.0:1788876690.899034:0:34846:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d6a.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788876690.899080:0:34846:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 34846 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.899088:0:34846:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34846 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.899098:0:34846:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 34846 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876690.899121:0:34846:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:34846:1875771511245696:192.168.203.111@tcp:101:checkstat.0 00000100:00100000:2.0:1788876690.899713:0:34846:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187aae840 going to sleep for 11 seconds 00000100:00100000:2.0:1788876690.909695:0:34846:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:34846:1875771511245696:192.168.203.111@tcp:101:checkstat.0 00010000:00010000:2.0:1788876690.910040:0:34846:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910068:0:34846:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910078:0:34846:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910093:0:34846:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910107:0:34846:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910124:0:34846:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910134:0:34846:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910148:0:34846:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876690.910163:0:34846:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888af800 x1875771511245696/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/688 e 0 to 0 dl 1788876706 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876690.910180:0:34846:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910192:0:34846:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/263 (0 0) 00000080:00200000:2.0:1788876690.910199:0:34846:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788876690.910230:0:34846:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x107:0x0], updating i_size 4096 00000080:00200000:2.0:1788876690.910236:0:34846:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x107:0x0](ffffa06198283248): rc = 0 00000080:00010000:2.0:1788876690.910243:0:34846:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x107:0x0](ffffa06198283248) for lock 0x7d14dcdc37048886 00000080:00010000:2.0:1788876690.910275:0:34846:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x107:0x0](ffffa06198283248) 00000080:00010000:2.0:1788876690.910281:0:34846:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048886 from it ffffb95ac0dd3b20 00010000:00010000:2.0:1788876690.910286:0:34846:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876690.910297:0:34846:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876690.910336:0:34846:0:(file.c:6296:ll_getattr_dentry()) START file d6a.replay-single:[0x200001b71:0x107:0x0](ffffa06198283248), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876690.910345:0:34846:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x107:0x0](ffffa06198283248),name=d6a.replay-single 00000002:00010000:2.0:1788876690.910369:0:34846:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x107:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876690.910441:0:34846:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876690.910457:0:34846:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876690.910466:0:34846:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876690.910472:0:34846:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x107:0x0](ffffa06198283248) 00000080:00010000:2.0:1788876690.910476:0:34846:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x107:0x0](ffffa06198283248) for lock 0x7d14dcdc37048886 00000080:00010000:2.0:1788876690.910481:0:34846:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048886 from it ffffb95ac0dd3cb8 00010000:00010000:2.0:1788876690.910485:0:34846:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876690.910493:0:34846:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876691.197555:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876693.630670:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511246080:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876693.632977:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511246336:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876693.633021:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1788876693.633028:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511246208:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876693.633050:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511246464:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876696.324564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876697.344592:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876698.779529:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511246720:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876698.779548:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511246976:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876698.779585:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511246592:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876698.779598:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511246848:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876698.779608:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511247104:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876700.417137:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876700.481810:0:35423:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 6b: |X| rmdir ====================== 10:11:38 (1788876698) 00000080:00200000:2.0:1788876700.692773:0:35431:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876700.692799:0:35431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876700.692820:0:35431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1788876700.692838:0:35431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1788876700.692847:0:35431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876700.692856:0:35431:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876700.692859:0:35431:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876700.692868:0:35431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac1d77b58 00010000:00010000:2.0:1788876700.692872:0:35431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1788876700.692882:0:35431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1788876700.692899:0:35431:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876700.692912:0:35431:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=13 00000080:00200000:2.0:1788876700.692916:0:35431:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:2.0:1788876700.692929:0:35431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d6b.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788876700.692976:0:35431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.692985:0:35431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.692995:0:35431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876700.693018:0:35431:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35431:1875771511247232:192.168.203.111@tcp:101:lfs.0 00000100:00100000:2.0:1788876700.693101:0:35431:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876700.705814:0:35431:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35431:1875771511247232:192.168.203.111@tcp:101:lfs.0 00010000:00010000:2.0:1788876700.705839:0:35431:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705855:0:35431:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705862:0:35431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705875:0:35431:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705884:0:35431:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705890:0:35431:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06180030800 already unused, calling callback (ffffffffc19a4630) 00010000:00010000:2.0:1788876700.705895:0:35431:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705902:0:35431:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705910:0:35431:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705918:0:35431:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876700.705926:0:35431:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876700.705933:0:35431:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048894 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 35431 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876700.705952:0:35431:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffffa061888af800 x1875771511247232/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788876716 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788876700.705967:0:35431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1788876700.705996:0:35431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1788876700.706009:0:35431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00010000:2.0:1788876700.706019:0:35431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac1d77930 00010000:00010000:2.0:1788876700.706023:0:35431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1788876700.706031:0:35431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1788876700.706101:0:35431:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876700.706116:0:35431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876700.706123:0:35431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1788876700.706132:0:35431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1788876700.706139:0:35431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876700.706144:0:35431:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876700.706147:0:35431:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876700.706151:0:35431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac1d77be8 00010000:00010000:2.0:1788876700.706154:0:35431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1788876700.706162:0:35431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1788876700.706171:0:35431:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876700.706177:0:35431:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6b.replay-single, flags=16392 00000080:00200008:2.0:1788876700.706200:0:35431:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876700.706205:0:35431:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876700.706212:0:35431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876700.706218:0:35431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1788876700.706227:0:35431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1788876700.706233:0:35431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876700.706237:0:35431:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876700.706240:0:35431:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876700.706243:0:35431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac1d77cb8 00010000:00010000:2.0:1788876700.706246:0:35431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1788876700.706253:0:35431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200008:2.0:1788876700.706978:0:35431:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876700.706983:0:35431:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876700.706992:0:35431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876700.707001:0:35431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1788876700.707011:0:35431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1788876700.707018:0:35431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876700.707024:0:35431:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876700.707026:0:35431:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876700.707030:0:35431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac1d77cb8 00010000:00010000:2.0:1788876700.707033:0:35431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1788876700.707040:0:35431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1788876700.707185:0:35431:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876700.707194:0:35431:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876700.707202:0:35431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1788876700.707212:0:35431:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:2.0:1788876700.707218:0:35431:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876700.707223:0:35431:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876700.707225:0:35431:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704887f 00000080:00010000:2.0:1788876700.707229:0:35431:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704887f from it ffffb95ac1d77bb0 00010000:00010000:2.0:1788876700.707233:0:35431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:2.0:1788876700.707241:0:35431:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1788876700.707251:0:35431:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876700.707258:0:35431:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 300000 00000080:00200080:2.0:1788876700.707274:0:35431:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c03066f0 arg=7ffd4af03fd0 00000080:00200000:2.0:1788876700.707288:0:35431:0:(dir.c:451:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) name=d6b.replay-single stripe_offset=0 stripe_count=1, hash_type=0 00010000:00010000:2.0:1788876700.707690:0:35431:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1788876700.707753:0:35431:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1788876700.707761:0:35431:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876700.707764:0:35431:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc3704887f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aee6a expref: -99 pid: 34362 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000020:00100000:2.0:1788876700.707790:0:35431:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876700.707802:0:35431:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888af800 x1875771511247488/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 568/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876700.707828:0:35431:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35431:1875771511247488:192.168.203.111@tcp:36:lfs.0 00000100:00100000:2.0:1788876700.707924:0:35431:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000080:00200008:2.0:1788876701.762271:0:35444:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876701.762285:0:35444:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876701.762308:0:35444:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876701.762396:0:35444:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35444 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.762407:0:35444:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35444 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.762419:0:35444:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35444 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876701.762456:0:35444:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511247616:192.168.203.111@tcp:101:lfs.0 00000100:00100000:2.0:1788876701.765607:0:35444:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876701.776812:0:35444:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 73014444033 gen 1 00000100:00100000:2.0:1788876701.776844:0:35444:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511247616:192.168.203.111@tcp:101:lfs.0 00010000:00010000:2.0:1788876701.776869:0:35444:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.776883:0:35444:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.776898:0:35444:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.776910:0:35444:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.776923:0:35444:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.776933:0:35444:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.776941:0:35444:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876701.776952:0:35444:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888ad180 x1875771511247616/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876717 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876701.776967:0:35444:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.776981:0:35444:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876701.776987:0:35444:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876701.777017:0:35444:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876701.777023:0:35444:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704889b 00000080:00010000:2.0:1788876701.777028:0:35444:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876701.777032:0:35444:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704889b from it ffffb95ac1e1fcb8 00010000:00010000:2.0:1788876701.777035:0:35444:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876701.777044:0:35444:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876701.778819:0:35444:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876701.778829:0:35444:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876701.778851:0:35444:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876701.778867:0:35444:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876701.778882:0:35444:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876701.778891:0:35444:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876701.778898:0:35444:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876701.778901:0:35444:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704889b 00000080:00010000:2.0:1788876701.778909:0:35444:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704889b from it ffffb95ac1e1fcb8 00010000:00010000:2.0:1788876701.778912:0:35444:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876701.778921:0:35444:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876701.779048:0:35444:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876701.779057:0:35444:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876701.779064:0:35444:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876701.779074:0:35444:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876701.779081:0:35444:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876701.779085:0:35444:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876701.779088:0:35444:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704889b 00000080:00010000:2.0:1788876701.779092:0:35444:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704889b from it ffffb95ac1e1fbb0 00010000:00010000:2.0:1788876701.779096:0:35444:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876701.779104:0:35444:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876701.779112:0:35444:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876701.779120:0:35444:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876701.779225:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.779242:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00800000:00000004:2.0:1788876701.779250:0:35444:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1682, max_age 1692 00000100:00100000:2.0:1788876701.779296:0:35444:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511247744:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876701.779405:0:35444:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876701.784838:0:35444:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511247744:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876701.784867:0:35444:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317387/353529 objects 1023705/1024000 00000080:00200080:2.0:1788876701.784934:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.784950:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00800000:00000004:2.0:1788876701.784958:0:35444:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1682, max_age 1692 00000100:00100000:2.0:1788876701.784992:0:35444:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511247872:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876701.785051:0:35444:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876701.796692:0:35444:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511247872:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876701.796722:0:35444:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:2.0:1788876701.796799:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796815:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796823:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796826:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796830:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796834:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796838:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796842:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796846:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796849:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796853:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796858:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796862:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796866:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796870:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796873:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796876:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796880:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796883:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796886:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796890:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796893:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796897:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796901:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796905:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796908:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796913:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796926:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796932:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796936:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796940:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796944:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796947:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796951:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796955:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796958:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796963:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796966:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796969:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796972:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796976:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796979:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796983:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796986:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796989:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796992:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.796995:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.796999:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797002:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797005:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797009:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797012:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797016:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797020:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797024:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797027:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797030:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797034:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797037:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797040:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797044:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797048:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797052:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797055:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797059:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797062:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797067:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797070:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797075:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797078:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797082:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797086:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797089:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797093:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797097:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797102:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797106:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797110:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797114:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797118:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797121:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797125:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797129:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797133:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797136:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797140:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797145:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797149:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797152:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797156:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797160:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797164:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797168:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797171:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797175:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797179:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797182:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797185:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797188:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797191:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797195:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797199:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797203:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797206:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797210:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797213:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797217:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797221:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797225:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797228:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797232:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797236:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797240:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797243:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797247:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797250:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797253:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797256:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797259:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797262:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797266:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797269:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797273:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797275:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797279:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00800000:00000080:2.0:1788876701.797282:0:35444:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788876701.797286:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:2.0:1788876701.797292:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00020000:00000004:2.0:1788876701.797298:0:35444:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1682, max_age 1692 00000100:00100000:2.0:1788876701.797333:0:35444:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511248000:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876701.797420:0:35444:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876701.799940:0:35444:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511248000:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876701.799969:0:35444:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876701.800092:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:2.0:1788876701.800111:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00020000:00000004:2.0:1788876701.800123:0:35444:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1682, max_age 1692 00000100:00100000:2.0:1788876701.800163:0:35444:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511248128:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876701.800227:0:35444:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c646c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876701.803767:0:35444:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35444:1875771511248128:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876701.803797:0:35444:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788876701.809199:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809227:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809241:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809246:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809249:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809253:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809256:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809259:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809263:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809266:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809270:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809273:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809277:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809279:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809283:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809286:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809291:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809294:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809297:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809301:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809305:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809308:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809313:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809316:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809319:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809323:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809326:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809330:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809334:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809338:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809342:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809345:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809349:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809352:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809356:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809359:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809363:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809367:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809371:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809374:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809378:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809407:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809412:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809417:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809422:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809426:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809430:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809433:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809437:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809441:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809456:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809461:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809466:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809470:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809474:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809477:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809481:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809485:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809489:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809492:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809497:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809501:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809505:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809508:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809512:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809516:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809520:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809524:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809528:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809532:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809536:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809540:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809544:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809548:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809552:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809556:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809561:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809564:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809568:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809572:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809576:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809579:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809583:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809587:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809592:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809596:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809599:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809603:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809620:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809626:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809631:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809635:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809639:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809642:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809645:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809649:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809653:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809656:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809660:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809663:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809667:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809670:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809675:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809678:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809682:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809686:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809691:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809694:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809697:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809701:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809705:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809708:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809712:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809716:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809721:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809724:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809729:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809732:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809736:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809740:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809744:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809748:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809753:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809756:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876701.809760:0:35444:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd066db2f0 00020000:00000080:3.0:1788876701.809764:0:35444:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200008:3.0:1788876701.809777:0:35444:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:1.0:1788876701.897502:0:35451:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876701.897529:0:35451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876701.897548:0:35451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876701.897563:0:35451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876701.897570:0:35451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876701.897577:0:35451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876701.897580:0:35451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704889b 00000080:00010000:1.0:1788876701.897587:0:35451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704889b from it ffffb95ac1da7bd0 00010000:00010000:1.0:1788876701.897591:0:35451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876701.897600:0:35451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1788876701.897616:0:35451:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876701.897630:0:35451:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:1.0:1788876701.897639:0:35451:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876701.897647:0:35451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876701.897655:0:35451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876701.897664:0:35451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1788876701.897671:0:35451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876701.897675:0:35451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876701.897678:0:35451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704889b 00000080:00010000:1.0:1788876701.897682:0:35451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704889b from it ffffb95ac1da7cd0 00010000:00010000:1.0:1788876701.897686:0:35451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876701.897693:0:35451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876701.897703:0:35451:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876701.897709:0:35451:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876701.897716:0:35451:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876701.897722:0:35451:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876701.897731:0:35451:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876701.897738:0:35451:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876701.897747:0:35451:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876701.897750:0:35451:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704889b 00000080:00010000:1.0:1788876701.897755:0:35451:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704889b from it ffffb95ac1da7db0 00010000:00010000:1.0:1788876701.897758:0:35451:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876701.897765:0:35451:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1788876701.897773:0:35451:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876701.897778:0:35451:0:(namei.c:2062:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:1.0:1788876701.897781:0:35451:0:(namei.c:2017:ll_mknod()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:1.0:1788876701.897806:0:35451:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1788876701.897856:0:35451:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1788876701.897864:0:35451:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788876701.897867:0:35451:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030200/0x7d14dcdc3704889b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aeeda expref: -99 pid: 35444 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:1.0:1788876701.897888:0:35451:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876701.897900:0:35451:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618869bb80 x1875771511248384/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876701.897923:0:35451:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:35451:1875771511248384:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:1.0:1788876701.897996:0:35451:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061995869c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876701.903934:0:35451:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:35451:1875771511248384:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:1.0:1788876701.903988:0:35451:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10b:0x0](ffffa061981c6c08) 00000080:00200000:1.0:1788876701.904027:0:35451:0:(namei.c:2070:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, unhashed 1 00000080:00200000:1.0:1788876701.915886:0:35454:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876701.915917:0:35454:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876701.915989:0:35454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35454 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.915999:0:35454:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35454 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.916009:0:35454:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35454 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876701.916034:0:35454:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35454:1875771511248512:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876701.916123:0:35454:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061995869c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876701.922161:0:35454:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35454:1875771511248512:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876701.922188:0:35454:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922200:0:35454:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922213:0:35454:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922224:0:35454:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922255:0:35454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922262:0:35454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922270:0:35454:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876701.922281:0:35454:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188698000 x1875771511248512/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876717 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876701.922305:0:35454:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922319:0:35454:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876701.922325:0:35454:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876701.922350:0:35454:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876701.922356:0:35454:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488a2 00000080:00010000:1.0:1788876701.922361:0:35454:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876701.922364:0:35454:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488a2 from it ffffb95ac1527b58 00010000:00010000:1.0:1788876701.922367:0:35454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922376:0:35454:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876701.922455:0:35454:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876701.922473:0:35454:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:1.0:1788876701.922478:0:35454:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:1.0:1788876701.922497:0:35454:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: fsa-oleg311-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788876701.922536:0:35454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35454 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922543:0:35454:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35454 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.922551:0:35454:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35454 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876701.922570:0:35454:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35454:1875771511248640:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876701.922627:0:35454:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061995869c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876701.929883:0:35454:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35454:1875771511248640:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876701.929911:0:35454:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.929935:0:35454:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.929942:0:35454:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.929955:0:35454:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.929979:0:35454:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.929993:0:35454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.930001:0:35454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.930008:0:35454:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876701.930019:0:35454:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188698000 x1875771511248640/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 600/584 e 0 to 0 dl 1788876717 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876701.930031:0:35454:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.930041:0:35454:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/267 (0 0) 00000080:00200000:1.0:1788876701.930047:0:35454:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876701.930067:0:35454:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10b:0x0], updating i_size 0 00000080:00200000:1.0:1788876701.930070:0:35454:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10b:0x0](ffffa061981c6c08): rc = 0 00000080:00200000:1.0:1788876701.930084:0:35454:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x10b:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1788876701.930089:0:35454:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10b:0x0](ffffa061981c6c08) for lock 0x7d14dcdc370488a9 00000080:00010000:1.0:1788876701.930099:0:35454:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10b:0x0](ffffa061981c6c08) 00000080:00010000:1.0:1788876701.930102:0:35454:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488a9 from it ffffb95ac1527b20 00010000:00010000:1.0:1788876701.930106:0:35454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876701.930121:0:35454:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1788876701.930158:0:35454:0:(file.c:6296:ll_getattr_dentry()) START file fsa-oleg311-client.virtnet:[0x200001b71:0x10b:0x0](ffffa061981c6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876701.930165:0:35454:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10b:0x0](ffffa061981c6c08),name=fsa-oleg311-client.virtnet 00000002:00010000:1.0:1788876701.930185:0:35454:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10b:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876701.930194:0:35454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876701.930205:0:35454:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876701.930212:0:35454:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876701.930216:0:35454:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10b:0x0](ffffa061981c6c08) 00000080:00010000:1.0:1788876701.930219:0:35454:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10b:0x0](ffffa061981c6c08) for lock 0x7d14dcdc370488a9 00000080:00010000:1.0:1788876701.930223:0:35454:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488a9 from it ffffb95ac1527cb8 00010000:00010000:1.0:1788876701.930227:0:35454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876701.930235:0:35454:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876701.930284:0:35454:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876701.930292:0:35454:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876701.930299:0:35454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876701.930313:0:35454:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876701.930320:0:35454:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876701.930325:0:35454:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876701.930327:0:35454:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488a2 00000080:00010000:1.0:1788876701.930332:0:35454:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488a2 from it ffffb95ac1527c10 00010000:00010000:1.0:1788876701.930335:0:35454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876701.930343:0:35454:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876701.930352:0:35454:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876701.930361:0:35454:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg311-client.virtnet, flags=0 00000080:00200000:1.0:1788876701.930366:0:35454:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876701.930372:0:35454:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876701.930378:0:35454:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876701.930424:0:35454:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876701.930431:0:35454:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876701.930436:0:35454:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876701.930438:0:35454:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488a2 00000080:00010000:1.0:1788876701.930442:0:35454:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488a2 from it ffffb95ac1527d68 00010000:00010000:1.0:1788876701.930461:0:35454:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876701.930469:0:35454:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876701.930479:0:35454:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876701.930485:0:35454:0:(namei.c:2351:ll_unlink()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876701.930506:0:35454:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876701.930528:0:35454:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876701.930539:0:35454:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x10b:0x0]: losing layout lock 00010000:00010000:1.0:1788876701.930570:0:35454:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876701.930578:0:35454:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876701.930585:0:35454:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876701.930588:0:35454:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fe00/0x7d14dcdc370488a9 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10b:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313aef12 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876701.930603:0:35454:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370488a2 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aeef6 expref: -99 pid: 35454 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788876701.930619:0:35454:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876701.930631:0:35454:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188698000 x1875771511248896/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876701.930654:0:35454:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35454:1875771511248896:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876701.930712:0:35454:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061995869c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876701.939595:0:35454:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35454:1875771511248896:192.168.203.111@tcp:36:rm.0 00000080:00200000:1.0:1788876701.939653:0:35454:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x10b:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:1.0:1788876701.939662:0:35454:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x10b:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1788876701.939669:0:35454:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10b:0x0](ffffa061981c6c08) 00010000:00010000:1.0:1788876703.485674:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876703.870960:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511249024:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.871097:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249280:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.873169:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249536:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.873268:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788876703.873284:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.873325:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249408:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.885642:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.885661:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249408:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.885673:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249280:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.885683:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511249536:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876703.885883:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511249024:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876706.557765:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876708.732898:0:35521:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:1.0:1788876708.760303:0:35522:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876708.760330:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876708.760429:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876708.760442:0:35522:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876708.760453:0:35522:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876708.760477:0:35522:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511249664:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876708.760552:0:35522:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061995869c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876708.769983:0:35522:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511249664:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876708.770017:0:35522:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770031:0:35522:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770044:0:35522:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770056:0:35522:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770071:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770080:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770088:0:35522:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876708.770102:0:35522:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869b100 x1875771511249664/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876724 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876708.770118:0:35522:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770134:0:35522:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876708.770142:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876708.770339:0:35522:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876708.770350:0:35522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488b0 00000080:00010000:3.0:1788876708.770356:0:35522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876708.770360:0:35522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488b0 from it ffffb95ac1c37b58 00010000:00010000:3.0:1788876708.770363:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770373:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876708.770543:0:35522:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876708.770570:0:35522:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:3.0:1788876708.770575:0:35522:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876708.770605:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d6b.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876708.770657:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770665:0:35522:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.770675:0:35522:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876708.770701:0:35522:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511249792:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876708.770803:0:35522:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06182bd1480 going to sleep for 11 seconds 00000100:00100000:3.0:1788876708.773700:0:35522:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511249792:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876708.773732:0:35522:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773757:0:35522:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773767:0:35522:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773780:0:35522:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773792:0:35522:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773806:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773816:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773824:0:35522:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876708.773838:0:35522:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188870380 x1875771511249792/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/688 e 0 to 0 dl 1788876724 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876708.773854:0:35522:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.773867:0:35522:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/266 (0 0) 00000080:00200000:3.0:1788876708.773874:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788876708.773979:0:35522:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10a:0x0], updating i_size 4096 00000080:00200000:3.0:1788876708.773984:0:35522:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10a:0x0](ffffa061981baa08): rc = 0 00000080:00010000:3.0:1788876708.773990:0:35522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10a:0x0](ffffa061981baa08) for lock 0x7d14dcdc370488b7 00000080:00010000:3.0:1788876708.774005:0:35522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10a:0x0](ffffa061981baa08) 00000080:00010000:3.0:1788876708.774009:0:35522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488b7 from it ffffb95ac1c37b20 00010000:00010000:3.0:1788876708.774013:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.774023:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876708.774071:0:35522:0:(file.c:6296:ll_getattr_dentry()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffffa061981baa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876708.774079:0:35522:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffffa061981baa08),name=d6b.replay-single 00000002:00010000:3.0:1788876708.774106:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876708.774116:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876708.774127:0:35522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876708.774134:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876708.774138:0:35522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10a:0x0](ffffa061981baa08) 00000080:00010000:3.0:1788876708.774142:0:35522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10a:0x0](ffffa061981baa08) for lock 0x7d14dcdc370488b7 00000080:00010000:3.0:1788876708.774145:0:35522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488b7 from it ffffb95ac1c37cb8 00010000:00010000:3.0:1788876708.774149:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876708.774158:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876708.774963:0:35522:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876708.774981:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876708.774994:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876708.775006:0:35522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876708.775013:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876708.775020:0:35522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876708.775023:0:35522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488b0 00000080:00010000:3.0:1788876708.775029:0:35522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488b0 from it ffffb95ac1c37bf8 00010000:00010000:3.0:1788876708.775032:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876708.775041:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876708.775051:0:35522:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876708.775061:0:35522:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6b.replay-single, flags=266 00000080:00200000:3.0:1788876708.775069:0:35522:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffffa061981baa08), inode mode 41ed mask 44 00000080:00200008:3.0:1788876708.775080:0:35522:0:(file.c:1090:ll_file_open()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffffa061981baa08), flags 704400 00000002:00010000:3.0:1788876708.775098:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240704401 00010000:00010000:3.0:1788876708.775153:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.775161:0:35522:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1788876708.775174:0:35522:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1788876708.775568:0:35522:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188870380 x1875771511250048/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1788876708.775580:0:35522:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876708.775610:0:35522:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250048:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876708.775813:0:35522:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06182bd1480 going to sleep for 11 seconds 00000100:00100000:3.0:1788876708.787528:0:35522:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250048:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876708.787558:0:35522:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.787574:0:35522:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.787582:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.787589:0:35522:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.787599:0:35522:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.788310:0:35522:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618887b800 already unused, calling callback (ffffffffc19a4630) 00010000:00010000:3.0:1788876708.788317:0:35522:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.788325:0:35522:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.788334:0:35522:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.788342:0:35522:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788876708.788349:0:35522:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876708.788357:0:35522:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370488be lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 35522 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876708.790514:0:35522:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa06188870380 x1875771511250048/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788876724 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1788876708.790605:0:35522:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188870380 x1875771511250048/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788876724 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1788876708.790625:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876708.790870:0:35522:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10a:0x0], updating i_size 4096 00000002:00100000:2.0:1788876708.790901:0:35522:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188870380 x1875771511250048/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/664 e 0 to 0 dl 1788876724 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1788876708.791015:0:35522:0:(file.c:6296:ll_getattr_dentry()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffffa061981baa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876708.791024:0:35522:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffffa061981baa08),name=d6b.replay-single 00000002:00010000:2.0:1788876708.791120:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876708.791212:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876708.791229:0:35522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876708.791238:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876708.791247:0:35522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10a:0x0](ffffa061981baa08) 00000080:00010000:2.0:1788876708.791251:0:35522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10a:0x0](ffffa061981baa08) for lock 0x7d14dcdc370488b7 00000080:00010000:2.0:1788876708.791259:0:35522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488b7 from it ffffb95ac1c37cd8 00010000:00010000:2.0:1788876708.791263:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876708.791273:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876708.791709:0:35522:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffffa061981baa08) pos/size0/4096 32bit_api 0 00000002:00010000:2.0:1788876708.791729:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10a:0x0]) in obj [0x200001b71:0x10a:0x0], intent: readdir flags 0 00010000:00010000:2.0:1788876708.791738:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876708.791747:0:35522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000100:00100000:2.0:1788876708.793298:0:35522:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250176:192.168.203.111@tcp:37:rm.0 00000100:00100000:2.0:1788876708.793797:0:35522:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64780 going to sleep for 11 seconds 00000100:00100000:2.0:1788876708.814983:0:35522:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250176:192.168.203.111@tcp:37:rm.0 00010000:00010000:2.0:1788876708.815770:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876708.815788:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876708.815854:0:35522:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffffa061981baa08) pos/size18446744073709551614/4096 32bit_api 0 00000080:00200008:2.0:1788876708.815971:0:35522:0:(file.c:479:ll_file_release()) START file d6b.replay-single:[0x200001b71:0x10a:0x0](ffffa061981baa08), flags 704000 00000002:00100000:2.0:1788876708.816055:0:35522:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa06188870380 x1875771511250048/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/664 e 0 to 0 dl 1788876724 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788876708.816067:0:35522:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788876708.816074:0:35522:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ad180 x1875771511250432/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788876708.816104:0:35522:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250432:192.168.203.111@tcp:35:rm.0 00000100:00100000:2.0:1788876708.816198:0:35522:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64780 going to sleep for 11 seconds 00000100:00100000:2.0:1788876708.832463:0:35522:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250432:192.168.203.111@tcp:35:rm.0 00000002:00100000:2.0:1788876708.832514:0:35522:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06188870380 x1875771511250048/t73014444036(73014444036) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/664 e 0 to 0 dl 1788876724 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788876708.832594:0:35522:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876708.832615:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876708.832638:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876708.832653:0:35522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876708.832663:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876708.832670:0:35522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876708.832674:0:35522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488b0 00000080:00010000:2.0:1788876708.832683:0:35522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488b0 from it ffffb95ac1c37c10 00010000:00010000:2.0:1788876708.832687:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876708.832696:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876708.832714:0:35522:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876708.832727:0:35522:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d6b.replay-single, flags=0 00000080:00200000:2.0:1788876708.832735:0:35522:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876708.832742:0:35522:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876708.832748:0:35522:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876708.832756:0:35522:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876708.832763:0:35522:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876708.832767:0:35522:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876708.832770:0:35522:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488b0 00000080:00010000:2.0:1788876708.832774:0:35522:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488b0 from it ffffb95ac1c37d88 00010000:00010000:2.0:1788876708.832778:0:35522:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876708.832786:0:35522:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876708.832795:0:35522:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876708.832803:0:35522:0:(namei.c:2261:ll_rmdir()) VFS Op:name=d6b.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:2.0:1788876708.832827:0:35522:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876708.832855:0:35522:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876708.832971:0:35522:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876708.832986:0:35522:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876708.832993:0:35522:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1788876708.832996:0:35522:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc370488b7 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10a:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aef58 expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876708.833013:0:35522:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e000/0x7d14dcdc370488b0 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aef3c expref: -99 pid: 35522 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000020:00100000:2.0:1788876708.833033:0:35522:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876708.833041:0:35522:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ad180 x1875771511250688/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876708.833066:0:35522:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250688:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876708.833162:0:35522:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64780 going to sleep for 11 seconds 00000100:00100000:2.0:1788876708.838702:0:35522:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:35522:1875771511250688:192.168.203.111@tcp:36:rm.0 00000080:00200000:2.0:1788876708.838760:0:35522:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10a:0x0](ffffa061981baa08) 00000100:00100000:1.0:1788876708.989714:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511250816:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876708.989833:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511251072:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876708.989893:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788876708.989899:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511250944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876708.989921:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511251200:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876708.991864:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511250816:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876709.629724:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876712.702029:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876714.114220:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511251328:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876714.114335:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251584:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876714.114378:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251840:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876714.114433:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876714.114441:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251456:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876714.114466:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251712:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876714.129894:0:2397:0:(recover.c:205:ptlrpc_request_handle_notconn()) import lustre-MDT0000-mdc-ffffa06189393000 of lustre-MDT0000_UUID@192.168.203.111@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1788876714.129904:0:2397:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1788876714.129910:0:2397:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa06189393000: Connection to lustre-MDT0000 (at 192.168.203.111@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1788876714.129918:0:2397:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876714.129923:0:2397:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1665 00000100:00080000:3.0:1788876714.129941:0:2397:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876714.129953:0:2397:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876714.129986:0:2397:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511251968/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876714.130020:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251456:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876714.130046:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251712:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876714.130062:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251584:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876714.131328:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511251840:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876714.131417:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511251968:192.168.203.111@tcp:38:ptlrpcd_01_00.0 00000100:00080000:2.0:1788876714.137955:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876714.137973:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876714.137981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511251968:192.168.203.111@tcp:38:ptlrpcd_01_00.0 00010000:00010000:1.0:1788876715.773666:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876718.845920:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876719.231454:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876719.231463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876719.231467:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876719.231481:0:8:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1705 00000100:00080000:3.0:1788876719.231493:0:8:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788876719.231501:0:8:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876719.231513:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876719.231534:0:8:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511252224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876719.231622:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876719.231738:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252608:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876719.231774:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1788876719.231782:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511252096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876719.231816:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252480:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876719.236355:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876719.242003:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252480:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876719.242024:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252608:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876721.919091:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788876724.349811:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876724.349820:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788876724.349824:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876724.349836:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1711 00000100:00080000:2.0:1788876724.349848:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1788876724.349855:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876724.349867:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876724.349886:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511252864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876724.349952:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252992:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876724.350054:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511253248:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876724.350090:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876724.350097:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511252736:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876724.350130:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511253120:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876724.350541:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511252864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876724.365377:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511253120:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876724.365418:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511252992:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876724.365430:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511253248:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876724.990535:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876728.061788:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788876729.470035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788876729.470044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788876729.470049:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788876729.470065:0:8:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1716 00000100:00080000:1.0:1788876729.470078:0:8:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1788876729.470086:0:8:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788876729.470107:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788876729.470131:0:8:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511253504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876729.470201:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511253632:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876729.470298:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511253888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876729.470343:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1788876729.470349:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511253376:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876729.470377:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511253760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876729.470984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511253504:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788876729.479062:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876729.479069:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876729.479073:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511253504:192.168.203.111@tcp:38:kworker.0 00000100:00000400:2.0:1788876730.493678:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876714/real 1788876714] req@ffffa061888af800 x1875771511251328/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876730 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876730.537153:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888af800 x1875771511251328/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876730 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876730.537172:0:2397:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1788876730.537177:0:2397:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.111@tcp: Connection to MGS (at 192.168.203.111@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1788876730.561640:0:2397:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.111@tcp for MGC192.168.203.111@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1788876730.561648:0:2397:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:2.0:1788876730.561658:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888af800 x1875771511251328/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876730 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876730.561672:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188871c00 x1875771511252096/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876735 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876730.561680:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888aed80 x1875771511252736/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876740 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876730.561695:0:2397:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188699c00 x1875771511253376/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876745 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1788876730.561717:0:2397:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1788876730.561721:0:2397:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1788876730.561722:0:2397:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:2.0:1788876730.561725:0:2397:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:2.0:1788876730.562584:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511252736:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876730.562601:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511252096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876730.562607:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511251328:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876730.563586:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511254400:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876730.563830:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511254016:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788876730.575818:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511254400:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876731.135204:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876734.205640:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876735.617572:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876735.617585:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788876735.617588:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876735.617592:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876735.617596:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876735.617609:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1722 00000100:00080000:3.0:1788876735.617622:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788876735.617630:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876735.617644:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876735.617685:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511254656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876735.617851:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511254784:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876735.618037:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511255040:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876735.629717:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511254912:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876735.655486:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511254784:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876735.655666:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511255040:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876735.655734:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876735.655748:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876735.655754:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511254656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876735.660937:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511254912:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876737.277855:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876740.734017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876740.734031:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788876740.734035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876740.734039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876740.734043:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876740.734055:0:37:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1727 00000100:00080000:3.0:1788876740.734077:0:37:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788876740.734088:0:37:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876740.734103:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876740.734146:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511255168/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876740.734298:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511255296:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876740.734495:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511255552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876740.734540:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1788876740.734560:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511255424:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876740.735326:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876740.735350:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788876740.735357:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511254016:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788876740.735729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876740.735735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788876740.735749:0:37:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876740.735755:0:37:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1722 00000100:00080000:3.0:1788876740.735762:0:37:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788876740.735768:0:37:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788876740.735797:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511255680/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876740.735854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511255680:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788876740.742617:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) MGC192.168.203.111@tcp: connect to target with instance 0 10000000:01000000:3.0:1788876740.742634:0:2394:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1788876740.742642:0:2394:0:(import.c:1328:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.203.111@tcp) after server handle changed from 0xf5bf0f7e313aeaea to 0xf5bf0f7e313af181 00010000:00010000:2.0:1788876740.745003:0:2397:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1788876740.745110:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511255296:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876740.762639:0:2397:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:2.0:1788876740.762656:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511255424:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876740.762682:0:2397:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 2 blwis in pool 00000100:00100000:2.0:1788876740.762687:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511255552:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876740.762846:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876740.768880:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1788876740.768905:0:2394:0:(import.c:1718:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.203.111@tcp; invalidating 00000100:00080000:1.0:1788876740.774550:0:35790:0:(import.c:1642:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.203.111@tcp to MGS@192.168.203.111@tcp 00000100:00100000:1.0:1788876740.774555:0:35790:0:(import.c:323:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:1.0:1788876740.774558:0:35790:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808003 00010000:00010000:1.0:1788876740.774574:0:35790:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313aeb7d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876740.774584:0:35790:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313aeb7d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876740.774591:0:35790:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1788876740.774611:0:35790:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876740.774621:0:35790:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc3704886a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313aeb7d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876740.774634:0:35790:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313aeb8b expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876740.774641:0:35790:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313aeb8b expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876740.774659:0:35790:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1788876740.774663:0:35790:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876740.774668:0:35790:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618902f200/0x7d14dcdc37048878 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313aeb8b expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876740.774682:0:35790:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313aeb76 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876740.774689:0:35790:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313aeb76 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876740.774694:0:35790:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:1.0:1788876740.774698:0:35790:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876740.774701:0:35790:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048863 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313aeb76 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876740.774714:0:35790:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313aeb84 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876740.774721:0:35790:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313aeb84 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876740.774725:0:35790:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:1.0:1788876740.774728:0:35790:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876740.774731:0:35790:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048871 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313aeb84 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1788876740.774744:0:35790:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:1.0:1788876740.774749:0:35790:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from RECOVER to FULL 10000000:01000000:1.0:1788876740.774757:0:35790:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:1.0:1788876740.774760:0:35790:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:1.0:1788876740.774765:0:35790:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) MGC192.168.203.111@tcp: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00000040:00080000:3.0:1788876742.393579:0:35791:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 51 00010000:00010000:2.0:1788876742.399518:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876742.433848:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618935f300 going to sleep for 11 seconds 00000100:00100000:2.0:1788876742.444268:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511256704:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000020:01000000:2.0:1788876742.445919:0:15194:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:2.0:1788876742.445947:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:2.0:1788876742.445961:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa061886ce000/0x7d14dcdc370488d3 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af196 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876742.445975:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa061886ce000/0x7d14dcdc370488d3 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af196 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876742.445988:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:2.0:1788876742.445992:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffffa0618906b380 from 1 10000000:01000000:2.0:1788876742.445995:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1788876742.446765:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876742.446780:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876742.446795:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876742.446829:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511256832:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1788876742.447016:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:2.0:1788876742.455506:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511256832:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876742.455544:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af19d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876742.455561:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313af19d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876742.455958:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511256960:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876742.456090:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64000 going to sleep for 11 seconds 00000100:00100000:2.0:1788876742.464374:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511256960:192.168.203.111@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1788876742.464460:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1788876742.464474:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af19d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876742.464487:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af19d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876743.424864:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:2.0:1788876744.706193:0:35803:0:(ldlm_request.c:2972:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffffa06189393000 to lustre-MDT0000_UUID@192.168.203.111@tcp 00010000:00010000:2.0:1788876744.706199:0:35803:0:(ldlm_request.c:2888:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffffa06189393000 (2) 00000080:00200000:2.0:1788876744.706245:0:35803:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x109:0x0]: losing layout lock 00010000:00010000:2.0:1788876744.706297:0:35803:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc3704888d lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x109:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0xf5bf0f7e313aeea2 expref: -99 pid: 34848 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876744.706334:0:35803:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048886 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x107:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0xf5bf0f7e313aee86 expref: -99 pid: 34846 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876744.706348:0:35803:0:(ldlm_request.c:2901:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffffa06189393000 00000100:00080000:2.0:1788876744.707700:0:35803:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00010000:00010000:2.0:1788876745.469740:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876748.460452:0:35847:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35847:1875771511258240:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876748.460485:0:35847:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876748.460540:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.460556:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00020000:00000004:2.0:1788876748.460563:0:35847:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1693, max_age 1739 00000100:00100000:2.0:1788876748.460594:0:35847:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35847:1875771511258368:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876748.460651:0:35847:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168f00 going to sleep for 11 seconds 00000100:00100000:2.0:1788876748.463661:0:35847:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:35847:1875771511258368:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876748.463706:0:35847:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788876748.463787:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463807:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463815:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463819:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463824:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463837:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463843:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463846:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463850:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463854:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463858:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463861:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463865:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463868:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463871:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463874:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463878:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463881:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463885:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463888:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463892:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463895:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463899:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463902:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463906:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463910:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463914:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463917:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463920:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463923:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463927:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.463931:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.463934:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464477:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464486:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464490:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464494:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464497:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464501:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464504:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464508:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464512:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464515:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464518:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464522:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464525:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464529:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464532:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464535:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464539:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464543:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464546:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464550:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464553:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464556:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464560:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464564:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464567:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464571:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464574:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464578:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464582:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464586:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464589:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464593:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464596:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464600:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464603:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464607:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464611:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464615:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464618:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464622:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464625:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464628:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464632:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464636:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464639:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464643:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464647:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464650:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464654:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464658:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464662:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464666:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464758:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464767:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464773:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464777:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464781:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464785:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464788:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464792:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464796:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464799:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464803:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464807:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464810:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464814:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464817:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464821:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464825:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464902:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464908:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464912:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464915:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464920:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464923:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464927:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464930:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.464934:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.464937:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.465009:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.465015:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.465019:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.465022:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.465026:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.465029:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.465033:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.465036:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.465040:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.465043:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.465047:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.465050:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200080:2.0:1788876748.465054:0:35847:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffa5cb0430 00020000:00000080:2.0:1788876748.465057:0:35847:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04800 uarg=0000000000000000 00000080:00200008:2.0:1788876748.465072:0:35847:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00010000:00010000:3.0:1788876748.553682:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876750.977721:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511258496:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.977820:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511258752:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.977849:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511259008:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.977878:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1788876750.977884:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511258624:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.977907:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511258880:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.986450:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511258624:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.986469:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511258880:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.986481:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511258752:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.986491:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511259008:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876750.989414:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511258496:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876751.615128:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876753.662137:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876754.686698:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876755.713101:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876756.097177:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511259136:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876756.097292:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259392:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876756.097321:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259648:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876756.097351:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1788876756.097357:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259264:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876756.097400:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259520:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876756.104644:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259264:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876756.104665:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259520:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876756.104676:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511259136:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876756.104688:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259392:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876756.104698:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511259648:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876756.742003:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876757.272225:0:36277:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:3.0:1788876757.759667:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876758.783253:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1788876759.489009:0:36316:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876759.489027:0:36316:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876759.489074:0:36316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876759.489107:0:36316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876759.489136:0:36316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876759.489147:0:36316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876759.489159:0:36316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876759.489166:0:36316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:2.0:1788876759.489177:0:36316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d87cb8 00010000:00010000:2.0:1788876759.489183:0:36316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876759.489195:0:36316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:2.0:1788876759.489753:0:36316:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876759.489759:0:36316:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876759.489767:0:36316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876759.489775:0:36316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876759.489784:0:36316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876759.489790:0:36316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876759.489794:0:36316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876759.489797:0:36316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:2.0:1788876759.489801:0:36316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d87cb8 00010000:00010000:2.0:1788876759.489805:0:36316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876759.489812:0:36316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876759.489893:0:36316:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876759.489902:0:36316:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876759.489909:0:36316:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876759.489917:0:36316:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876759.489924:0:36316:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876759.489928:0:36316:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876759.489930:0:36316:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:2.0:1788876759.489934:0:36316:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d87bb0 00010000:00010000:2.0:1788876759.489937:0:36316:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876759.489945:0:36316:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1788876759.489954:0:36316:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876759.489961:0:36316:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876759.493244:0:36316:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffea7cb3990 00800000:00000080:2.0:1788876759.493274:0:36316:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00800000:00000004:2.0:1788876759.493284:0:36316:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1740, max_age 1750 00000100:00100000:2.0:1788876759.493337:0:36316:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36316:1875771511259776:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876759.493543:0:36316:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06182e05000 going to sleep for 11 seconds 00000100:00100000:2.0:1788876759.507692:0:36316:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36316:1875771511259776:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876759.507723:0:36316:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023706/1024000 00000080:00200080:2.0:1788876759.507866:0:36316:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffea7cb3990 00800000:00000080:2.0:1788876759.507884:0:36316:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00800000:00000004:2.0:1788876759.507891:0:36316:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1740, max_age 1750 00000100:00100000:2.0:1788876759.507927:0:36316:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36316:1875771511259904:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876759.508011:0:36316:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06182e05000 going to sleep for 11 seconds 00000100:00100000:2.0:1788876759.526325:0:36316:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36316:1875771511260032:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876759.526358:0:36316:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876759.526441:0:36316:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffea7cb3990 00020000:00000080:2.0:1788876759.526459:0:36316:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e04400 uarg=0000000000000000 00020000:00000004:2.0:1788876759.526476:0:36316:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1740, max_age 1750 00000100:00100000:2.0:1788876759.526516:0:36316:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36316:1875771511260160:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876759.526580:0:36316:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06182e05cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876759.538280:0:36316:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36316:1875771511260160:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876759.538314:0:36316:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00010000:00010000:1.0:1788876759.608501:0:36332:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1788876759.608524:0:36332:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00010000:1.0:1788876759.608537:0:36332:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0c87930 00010000:00010000:1.0:1788876759.608543:0:36332:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1788876759.608553:0:36332:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1788876759.807574:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876760.831583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876761.213725:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511260416:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876761.213867:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260672:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876761.213899:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260928:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876761.213928:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788876761.213934:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260544:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876761.213957:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260800:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876761.219442:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260544:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876761.219460:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260800:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876761.219470:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511260416:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876761.219481:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260672:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876761.219490:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511260928:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876761.860797:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876763.908668:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876764.930276:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876766.354165:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511261184:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876766.354183:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511261440:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876766.354193:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511261056:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876766.354204:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511261312:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876766.363082:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511261568:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876766.982351:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1788876767.006173:0:36906:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 7: mkdir |X| contained create ====== 10:12:45 (1788876765) 00000080:00200000:3.0:1788876767.309780:0:36918:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876767.309809:0:36918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876767.310610:0:36918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1788876767.310633:0:36918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:3.0:1788876767.310642:0:36918:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876767.310652:0:36918:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876767.310655:0:36918:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:3.0:1788876767.310663:0:36918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d7fb58 00010000:00010000:3.0:1788876767.310667:0:36918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1788876767.310677:0:36918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1788876767.310693:0:36918:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876767.310711:0:36918:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=13 00000080:00200000:3.0:1788876767.310715:0:36918:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876767.310730:0:36918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876767.310776:0:36918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.310784:0:36918:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.310794:0:36918:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876767.310891:0:36918:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36918:1875771511261696:192.168.203.111@tcp:101:lfs.0 00000100:00100000:3.0:1788876767.311259:0:36918:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c13c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876767.323188:0:36918:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36918:1875771511261696:192.168.203.111@tcp:101:lfs.0 00010000:00010000:3.0:1788876767.323214:0:36918:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323229:0:36918:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323236:0:36918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323244:0:36918:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323252:0:36918:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323259:0:36918:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618887b400 already unused, calling callback (ffffffffc19a4630) 00010000:00010000:3.0:1788876767.323264:0:36918:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323270:0:36918:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323278:0:36918:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323285:0:36918:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788876767.323293:0:36918:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876767.323304:0:36918:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370488ef lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 36918 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876767.323323:0:36918:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffffa06188872300 x1875771511261696/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788876783 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'lfs.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1788876767.323339:0:36918:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1788876767.323366:0:36918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1788876767.323411:0:36918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00010000:3.0:1788876767.323422:0:36918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d7f930 00010000:00010000:3.0:1788876767.323426:0:36918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1788876767.323434:0:36918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1788876767.323507:0:36918:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876767.323521:0:36918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876767.323529:0:36918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1788876767.323538:0:36918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1788876767.323656:0:36918:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876767.323661:0:36918:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876767.323665:0:36918:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:3.0:1788876767.323670:0:36918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d7fbe8 00010000:00010000:3.0:1788876767.323674:0:36918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1788876767.323682:0:36918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1788876767.323692:0:36918:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876767.323700:0:36918:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=16392 00000080:00200008:3.0:1788876767.323725:0:36918:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1788876767.323730:0:36918:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876767.323736:0:36918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876767.323743:0:36918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1788876767.323752:0:36918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1788876767.323758:0:36918:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876767.323762:0:36918:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876767.323764:0:36918:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:3.0:1788876767.323767:0:36918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d7fcb8 00010000:00010000:3.0:1788876767.323770:0:36918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1788876767.323778:0:36918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200008:3.0:1788876767.324368:0:36918:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1788876767.324374:0:36918:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876767.324483:0:36918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876767.324493:0:36918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:3.0:1788876767.324504:0:36918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:3.0:1788876767.324510:0:36918:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876767.324515:0:36918:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876767.324518:0:36918:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:3.0:1788876767.324523:0:36918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d7fcb8 00010000:00010000:3.0:1788876767.324526:0:36918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:3.0:1788876767.324534:0:36918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1788876767.324720:0:36918:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876767.324729:0:36918:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876767.324736:0:36918:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1788876767.324751:0:36918:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:3.0:1788876767.324757:0:36918:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876767.324761:0:36918:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876767.324764:0:36918:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488e1 00000080:00010000:3.0:1788876767.324768:0:36918:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488e1 from it ffffb95ac0d7fbb0 00010000:00010000:3.0:1788876767.324771:0:36918:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:3.0:1788876767.324778:0:36918:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:3.0:1788876767.324787:0:36918:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:3.0:1788876767.324794:0:36918:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 300000 00000080:00200080:3.0:1788876767.324810:0:36918:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c03066f0 arg=7ffeeb7808f0 00000080:00200000:3.0:1788876767.324821:0:36918:0:(dir.c:451:ll_dir_setdirstripe()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) name=d7.replay-single stripe_offset=0 stripe_count=1, hash_type=0 00010000:00010000:3.0:1788876767.324845:0:36918:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1788876767.324900:0:36918:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:3.0:1788876767.324907:0:36918:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876767.324910:0:36918:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce400/0x7d14dcdc370488e1 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af3bf expref: -99 pid: 35847 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000020:00100000:3.0:1788876767.325028:0:36918:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876767.325038:0:36918:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188872300 x1875771511261952/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 568/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876767.325061:0:36918:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36918:1875771511261952:192.168.203.111@tcp:36:lfs.0 00000100:00100000:3.0:1788876767.325204:0:36918:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c13c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876767.342110:0:36918:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36918:1875771511261952:192.168.203.111@tcp:36:lfs.0 00000080:00200000:2.0:1788876767.345029:0:36918:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00200000:2.0:1788876767.345042:0:36918:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 4096 00000080:00200008:2.0:1788876767.345086:0:36918:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 300000 00010000:00010000:1.0:1788876768.001068:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1788876768.279084:0:36927:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876768.279100:0:36927:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876768.279126:0:36927:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876768.279197:0:36927:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 36927 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876768.279208:0:36927:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 36927 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876768.279219:0:36927:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 36927 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876768.279247:0:36927:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262080:192.168.203.111@tcp:101:lfs.0 00000100:00100000:1.0:1788876768.279477:0:36927:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618600c9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876768.288906:0:36927:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 77309411329 gen 1 00000100:00100000:2.0:1788876768.288936:0:36927:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262080:192.168.203.111@tcp:101:lfs.0 00010000:00010000:2.0:1788876768.288960:0:36927:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.288985:0:36927:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.289000:0:36927:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.289013:0:36927:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.289027:0:36927:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.289036:0:36927:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.289050:0:36927:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876768.289063:0:36927:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869ad80 x1875771511262080/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876784 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876768.289090:0:36927:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.289104:0:36927:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876768.289112:0:36927:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876768.289139:0:36927:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876768.289145:0:36927:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488f6 00000080:00010000:2.0:1788876768.289150:0:36927:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.289154:0:36927:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488f6 from it ffffb95ac1557cb8 00010000:00010000:2.0:1788876768.289157:0:36927:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.289167:0:36927:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876768.289873:0:36927:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876768.289880:0:36927:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876768.289897:0:36927:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.289908:0:36927:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876768.289927:0:36927:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876768.289934:0:36927:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.289940:0:36927:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.289943:0:36927:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488f6 00000080:00010000:2.0:1788876768.289948:0:36927:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488f6 from it ffffb95ac1557cb8 00010000:00010000:2.0:1788876768.289951:0:36927:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876768.289960:0:36927:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876768.290083:0:36927:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876768.290093:0:36927:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.290100:0:36927:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876768.290108:0:36927:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876768.290114:0:36927:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.290118:0:36927:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.290121:0:36927:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488f6 00000080:00010000:2.0:1788876768.290124:0:36927:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488f6 from it ffffb95ac1557bb0 00010000:00010000:2.0:1788876768.290127:0:36927:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876768.290135:0:36927:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876768.290144:0:36927:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876768.290152:0:36927:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876768.290238:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.290253:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00800000:00000004:2.0:1788876768.290259:0:36927:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1751, max_age 1759 00000100:00100000:2.0:1788876768.290295:0:36927:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262208:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876768.290367:0:36927:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788876768.295806:0:36927:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262208:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876768.295836:0:36927:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317387/353529 objects 1023705/1024000 00000080:00200080:2.0:1788876768.297394:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.297418:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00800000:00000004:2.0:1788876768.297426:0:36927:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1751, max_age 1759 00000100:00100000:2.0:1788876768.297468:0:36927:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262336:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876768.297553:0:36927:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788876768.315631:0:36927:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262336:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876768.315663:0:36927:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:2.0:1788876768.315732:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315750:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315759:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315763:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315768:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315771:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315774:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315777:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315781:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315784:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315788:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315791:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315794:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315797:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315801:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315805:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315808:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315812:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315816:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315819:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315823:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315827:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315831:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315835:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315839:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315842:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315847:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315850:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315854:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315858:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315861:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315864:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315868:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315871:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315875:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315879:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315883:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315886:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315890:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315894:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315898:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315901:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315905:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315909:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315912:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315916:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315919:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315923:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315927:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315931:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315934:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315938:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315960:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315966:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315972:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315985:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315990:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.315994:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.315998:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316002:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316006:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316010:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316014:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316018:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316022:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316026:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316030:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316033:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316036:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316040:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316043:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316047:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316051:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316055:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316059:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.316062:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.316066:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:2.0:1788876768.317299:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323775:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323805:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323826:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323831:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323835:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323839:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323843:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323846:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323850:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323854:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323858:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323861:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323865:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323868:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323871:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323874:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323878:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323883:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323886:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323889:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323893:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323897:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323901:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323904:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323907:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323910:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323914:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323918:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323921:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323925:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323928:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323931:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323935:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323938:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323947:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323950:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323954:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323958:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323961:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323964:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323968:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323971:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323987:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323991:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.323996:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.323999:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.324003:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00800000:00000080:3.0:1788876768.324006:0:36927:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00000080:00200080:3.0:1788876768.324010:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:3.0:1788876768.324016:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188388c00 uarg=0000000000000000 00020000:00000004:3.0:1788876768.324023:0:36927:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1751, max_age 1759 00000100:00100000:3.0:1788876768.324300:0:36927:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262464:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876768.324457:0:36927:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c13c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876768.337770:0:36927:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262464:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876768.337807:0:36927:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876768.344894:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.344923:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00020000:00000004:2.0:1788876768.344933:0:36927:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1751, max_age 1759 00000100:00100000:2.0:1788876768.344994:0:36927:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262592:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876768.345100:0:36927:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788876768.353008:0:36927:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:36927:1875771511262592:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876768.353039:0:36927:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788876768.353108:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353126:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353133:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353136:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353141:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353144:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353147:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353150:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353154:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353158:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353162:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353165:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353169:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353172:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353176:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353179:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353183:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353186:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353192:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353196:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353200:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353202:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353206:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353210:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353214:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353217:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353431:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353438:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353443:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353447:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353450:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353453:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353457:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353460:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353464:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353468:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353471:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353475:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353479:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353482:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353487:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353491:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353495:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353499:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353503:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353507:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353511:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353514:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353519:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353523:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353527:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353531:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353534:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353538:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353542:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353546:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353550:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353553:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353557:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353561:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353565:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353568:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353572:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353574:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353577:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353581:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353584:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353586:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353590:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353594:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353597:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353600:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353604:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353606:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353610:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353613:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353616:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353620:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353623:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353626:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353630:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353633:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353637:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353640:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353643:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353646:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353649:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353653:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353657:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353660:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353664:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353667:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353670:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353674:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353677:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353680:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353684:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353954:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353962:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353966:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353970:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353982:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353987:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353990:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.353994:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.353998:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354001:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354005:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354009:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354012:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354015:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354019:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354023:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354026:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354030:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354033:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354037:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354040:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354045:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354048:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354052:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354055:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354058:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354061:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200080:2.0:1788876768.354065:0:36927:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc31c4c090 00020000:00000080:2.0:1788876768.354067:0:36927:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05000 uarg=0000000000000000 00000080:00200008:2.0:1788876768.354081:0:36927:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:2.0:1788876768.466061:0:36934:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876768.466091:0:36934:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.466112:0:36934:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876768.466129:0:36934:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876768.466137:0:36934:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.466144:0:36934:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.466147:0:36934:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488f6 00000080:00010000:2.0:1788876768.466154:0:36934:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488f6 from it ffffb95ac1e17bd0 00010000:00010000:2.0:1788876768.466159:0:36934:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876768.466168:0:36934:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876768.466190:0:36934:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876768.466205:0:36934:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:2.0:1788876768.466209:0:36934:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876768.466213:0:36934:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.466218:0:36934:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876768.466226:0:36934:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876768.466232:0:36934:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.466235:0:36934:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.466237:0:36934:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488f6 00000080:00010000:2.0:1788876768.466240:0:36934:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488f6 from it ffffb95ac1e17cd0 00010000:00010000:2.0:1788876768.466243:0:36934:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876768.466249:0:36934:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876768.466257:0:36934:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876768.466262:0:36934:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876768.466265:0:36934:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.466270:0:36934:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876768.466282:0:36934:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876768.466288:0:36934:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.466291:0:36934:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.466293:0:36934:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488f6 00000080:00010000:2.0:1788876768.466296:0:36934:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488f6 from it ffffb95ac1e17db0 00010000:00010000:2.0:1788876768.466299:0:36934:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876768.466305:0:36934:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1788876768.466312:0:36934:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876768.466317:0:36934:0:(namei.c:2062:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:2.0:1788876768.466320:0:36934:0:(namei.c:2017:ll_mknod()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:2.0:1788876768.466348:0:36934:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876768.466426:0:36934:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876768.466433:0:36934:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876768.466436:0:36934:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f200/0x7d14dcdc370488f6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af40c expref: -99 pid: 36927 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:2.0:1788876768.466460:0:36934:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876768.466471:0:36934:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ae680 x1875771511262848/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876768.466497:0:36934:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:36934:1875771511262848:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876768.466591:0:36934:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788876768.486471:0:36934:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:36934:1875771511262848:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:2.0:1788876768.486527:0:36934:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10d:0x0](ffffa0619828aa08) 00000080:00200000:2.0:1788876768.486566:0:36934:0:(namei.c:2070:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, unhashed 1 00000080:00200000:1.0:1788876768.508145:0:36937:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876768.508176:0:36937:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876768.508244:0:36937:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 36937 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876768.508256:0:36937:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 36937 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876768.508268:0:36937:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 36937 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876768.508294:0:36937:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:36937:1875771511262976:192.168.203.111@tcp:101:rm.0 00000100:00100000:1.0:1788876768.508599:0:36937:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618600c300 going to sleep for 11 seconds 00000100:00100000:3.0:1788876768.522226:0:36937:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:36937:1875771511262976:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876768.522261:0:36937:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876768.522273:0:36937:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876768.522286:0:36937:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876768.522299:0:36937:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876768.522319:0:36937:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876768.522327:0:36937:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876768.522337:0:36937:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876768.522349:0:36937:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869b800 x1875771511262976/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876784 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876768.522366:0:36937:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876768.522536:0:36937:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876768.522543:0:36937:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876768.523055:0:36937:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876768.523064:0:36937:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488fd 00000080:00010000:2.0:1788876768.523069:0:36937:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.523073:0:36937:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488fd from it ffffb95ac1557b58 00010000:00010000:2.0:1788876768.523076:0:36937:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.523085:0:36937:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876768.523113:0:36937:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876768.523136:0:36937:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:2.0:1788876768.523140:0:36937:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:2.0:1788876768.523162:0:36937:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: fsa-oleg311-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788876768.523210:0:36937:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 36937 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.523217:0:36937:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 36937 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.523230:0:36937:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 36937 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876768.523297:0:36937:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:36937:1875771511263104:192.168.203.111@tcp:101:rm.0 00000100:00100000:2.0:1788876768.524005:0:36937:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788876768.528004:0:36937:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:36937:1875771511263104:192.168.203.111@tcp:101:rm.0 00010000:00010000:2.0:1788876768.528031:0:36937:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528052:0:36937:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528060:0:36937:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528071:0:36937:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528083:0:36937:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528096:0:36937:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528103:0:36937:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528114:0:36937:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876768.528125:0:36937:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888afb80 x1875771511263104/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 600/584 e 0 to 0 dl 1788876784 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876768.528136:0:36937:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528146:0:36937:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/269 (0 0) 00000080:00200000:2.0:1788876768.528151:0:36937:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788876768.528170:0:36937:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10d:0x0], updating i_size 0 00000080:00200000:2.0:1788876768.528173:0:36937:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10d:0x0](ffffa0619828aa08): rc = 0 00000080:00200000:2.0:1788876768.528188:0:36937:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x10d:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1788876768.528193:0:36937:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10d:0x0](ffffa0619828aa08) for lock 0x7d14dcdc37048904 00000080:00010000:2.0:1788876768.528203:0:36937:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10d:0x0](ffffa0619828aa08) 00000080:00010000:2.0:1788876768.528207:0:36937:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048904 from it ffffb95ac1557b20 00010000:00010000:2.0:1788876768.528210:0:36937:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876768.528219:0:36937:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876768.528256:0:36937:0:(file.c:6296:ll_getattr_dentry()) START file fsa-oleg311-client.virtnet:[0x200001b71:0x10d:0x0](ffffa0619828aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876768.528262:0:36937:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10d:0x0](ffffa0619828aa08),name=fsa-oleg311-client.virtnet 00000002:00010000:2.0:1788876768.528282:0:36937:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10d:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.528292:0:36937:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876768.528304:0:36937:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876768.528311:0:36937:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.528315:0:36937:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10d:0x0](ffffa0619828aa08) 00000080:00010000:2.0:1788876768.528318:0:36937:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10d:0x0](ffffa0619828aa08) for lock 0x7d14dcdc37048904 00000080:00010000:2.0:1788876768.528322:0:36937:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048904 from it ffffb95ac1557cb8 00010000:00010000:2.0:1788876768.528325:0:36937:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876768.528332:0:36937:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876768.528827:0:36937:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876768.528837:0:36937:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.528846:0:36937:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876768.528855:0:36937:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876768.528862:0:36937:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.528866:0:36937:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.528869:0:36937:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488fd 00000080:00010000:2.0:1788876768.528872:0:36937:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488fd from it ffffb95ac1557c10 00010000:00010000:2.0:1788876768.528875:0:36937:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876768.528883:0:36937:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876768.528892:0:36937:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876768.528900:0:36937:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg311-client.virtnet, flags=0 00000080:00200000:2.0:1788876768.528905:0:36937:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876768.528910:0:36937:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876768.528915:0:36937:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876768.528922:0:36937:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876768.528927:0:36937:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876768.528930:0:36937:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876768.528933:0:36937:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370488fd 00000080:00010000:2.0:1788876768.528936:0:36937:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370488fd from it ffffb95ac1557d68 00010000:00010000:2.0:1788876768.528939:0:36937:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876768.528946:0:36937:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876768.528953:0:36937:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876768.528958:0:36937:0:(namei.c:2351:ll_unlink()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:2.0:1788876768.528988:0:36937:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876768.529011:0:36937:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876768.529020:0:36937:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x10d:0x0]: losing layout lock 00010000:00010000:2.0:1788876768.529055:0:36937:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876768.529062:0:36937:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876768.529068:0:36937:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1788876768.529071:0:36937:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc37048904 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10d:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313af444 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876768.529088:0:36937:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc370488fd lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af428 expref: -99 pid: 36937 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1788876768.529104:0:36937:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876768.529112:0:36937:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888afb80 x1875771511263360/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876768.529133:0:36937:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:36937:1875771511263360:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876768.529208:0:36937:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168d80 going to sleep for 11 seconds 00000100:00100000:2.0:1788876768.532525:0:36937:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:36937:1875771511263360:192.168.203.111@tcp:36:rm.0 00000080:00200000:2.0:1788876768.532576:0:36937:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x10d:0x0] ignore/verify layout 1/0, layout version 65535 restore needed 0 00000080:00200000:2.0:1788876768.532585:0:36937:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x10d:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788876768.532594:0:36937:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10d:0x0](ffffa0619828aa08) 00010000:00010000:2.0:1788876770.045647:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876771.070942:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876771.454226:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511263488:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876771.454348:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511263744:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876771.454408:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511264000:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876771.454443:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1788876771.454450:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511263616:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876771.454472:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511263872:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876773.123982:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876774.141634:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876775.316180:0:37004:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:3.0:1788876775.334311:0:37005:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876775.334340:0:37005:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876775.336033:0:37005:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 37005 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.336047:0:37005:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37005 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.336060:0:37005:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 37005 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876775.336086:0:37005:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:37005:1875771511264128:192.168.203.111@tcp:101:mcreate.0 00000100:00100000:3.0:1788876775.336208:0:37005:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c1cc0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876775.341258:0:37005:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:37005:1875771511264128:192.168.203.111@tcp:101:mcreate.0 00010000:00010000:3.0:1788876775.341287:0:37005:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341300:0:37005:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341314:0:37005:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341325:0:37005:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341337:0:37005:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341346:0:37005:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341354:0:37005:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876775.341365:0:37005:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188872300 x1875771511264128/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876791 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876775.341379:0:37005:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341435:0:37005:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876775.341442:0:37005:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876775.341469:0:37005:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876775.341475:0:37005:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704890b 00000080:00010000:3.0:1788876775.341481:0:37005:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876775.341483:0:37005:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704890b from it ffffb95ac1c9fbd0 00010000:00010000:3.0:1788876775.341486:0:37005:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341495:0:37005:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876775.341529:0:37005:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876775.341548:0:37005:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=24 00000080:00200000:3.0:1788876775.341553:0:37005:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=0 00000002:00010000:3.0:1788876775.341575:0:37005:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: lookup flags 0 00010000:00010000:3.0:1788876775.341625:0:37005:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37005 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341634:0:37005:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37005 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876775.341846:0:37005:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37005 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876775.341934:0:37005:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:37005:1875771511264256:192.168.203.111@tcp:101:mcreate.0 00000100:00100000:2.0:1788876775.342001:0:37005:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061891680c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876775.358986:0:37005:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:37005:1875771511264256:192.168.203.111@tcp:101:mcreate.0 00010000:00010000:2.0:1788876775.359017:0:37005:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359041:0:37005:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359050:0:37005:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359063:0:37005:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359075:0:37005:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359088:0:37005:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359098:0:37005:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359112:0:37005:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876775.359124:0:37005:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffffa06188872300 x1875771511264256/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/688 e 0 to 0 dl 1788876791 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876775.359139:0:37005:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359150:0:37005:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/268 (0 0) 00000080:00200000:2.0:1788876775.359156:0:37005:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788876775.359184:0:37005:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 4096 00000080:00200000:2.0:1788876775.359187:0:37005:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10c:0x0](ffffa0619828db88): rc = 0 00000080:00010000:2.0:1788876775.359192:0:37005:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc37048912 00000080:00010000:2.0:1788876775.359203:0:37005:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00010000:2.0:1788876775.359207:0:37005:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048912 from it ffffb95ac1c9faa0 00010000:00010000:2.0:1788876775.359210:0:37005:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876775.359220:0:37005:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876775.359252:0:37005:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 1 00000080:00200000:2.0:1788876775.359271:0:37005:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffffa0619828db88), flags=1536 00000080:00200000:2.0:1788876775.359275:0:37005:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 3 00000080:00200000:2.0:1788876775.359280:0:37005:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 3 00000080:00200000:2.0:1788876775.359285:0:37005:0:(namei.c:2062:ll_create_nd()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffffa0619828db88), flags=33188, excl=1 00000080:00200000:2.0:1788876775.359289:0:37005:0:(namei.c:2017:ll_mknod()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffffa0619828db88) mode 100644 dev 0 00000020:00100000:2.0:1788876775.359345:0:37005:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876775.359352:0:37005:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ae300 x1875771511264512/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876775.359373:0:37005:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:37005:1875771511264512:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876775.359525:0:37005:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061891680c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876775.365360:0:37005:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:37005:1875771511264512:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:2.0:1788876775.365457:0:37005:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x10e:0x0](ffffa0619828aa08) 00000080:00200000:2.0:1788876775.365496:0:37005:0:(namei.c:2070:ll_create_nd()) VFS Op:name=f7.replay-single, unhashed 1 00010000:00010000:2.0:1788876776.189571:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876776.573727:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511264768:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.573862:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511265024:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.573893:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1788876776.573899:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511264640:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.573938:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511264896:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.573960:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511265152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.588901:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511265024:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.588945:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511264640:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.588957:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511264896:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.588968:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511265152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876776.588983:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511264768:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876777.213648:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876778.237633:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876779.261751:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876780.294198:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876781.309833:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876781.702316:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511265920:192.168.203.111@tcp:38:ptlrpcd_00_01.0 00000100:00080000:3.0:1788876781.725025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876781.725037:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876781.725042:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511265920:192.168.203.111@tcp:38:ptlrpcd_00_01.0 00010000:00010000:2.0:1788876782.335239:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876783.358771:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876784.381679:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876785.405788:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876786.429855:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876786.814196:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511266304:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876786.814325:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511266560:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876786.814362:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788876786.814369:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511266048:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876786.814446:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511266432:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876786.814928:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511266176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788876786.820969:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511266304:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876786.839088:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876786.839106:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876786.839113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511266176:192.168.203.111@tcp:38:kworker.0 00010000:00010000:1.0:1788876787.453677:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876788.477807:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876789.502190:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876790.525735:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876791.550595:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788876791.933941:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876791.933950:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788876791.933954:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876791.933967:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1778 00000100:00080000:2.0:1788876791.933978:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1788876791.933986:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876791.933998:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876791.934017:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511266816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876791.934992:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511266688:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876791.935055:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267072:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876791.935083:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788876791.935090:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511266944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876791.935110:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267200:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876791.948563:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511266944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876791.948585:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267200:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876791.948597:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267072:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876792.575562:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876793.598287:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876794.621623:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876795.648113:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876796.669627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1788876797.053477:0:2396:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876781/real 1788876781] req@ffffa06198aff100 x1875771511265280/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876797 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788876797.053518:0:2396:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06198aff100 x1875771511265280/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876797 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788876797.053528:0:2396:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from FULL to DISCONN 00000100:00080000:2.0:1788876797.053531:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:02020000:1.0:1788876797.053534:0:2396:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.111@tcp: Connection to MGS (at 192.168.203.111@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1788876797.053539:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788876797.053542:0:2396:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.111@tcp for MGC192.168.203.111@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1788876797.053544:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788876797.053546:0:2396:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00080000:2.0:1788876797.053555:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1783 00000100:00100000:1.0:1788876797.053556:0:2396:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06198aff100 x1875771511265280/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876797 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876797.053568:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00100000:1.0:1788876797.053570:0:2396:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06198afd880 x1875771511266048/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876802 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876797.053580:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788876797.053595:0:2396:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888ac380 x1875771511266688/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876807 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788876797.053595:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 10000000:01000000:1.0:1788876797.053612:0:2396:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1788876797.053615:0:2396:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1788876797.053616:0:2396:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1788876797.053618:0:2396:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:2.0:1788876797.053641:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511267456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876797.053642:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511265280:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876797.053701:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511266048:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876797.053718:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876797.053723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788876797.053726:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876797.053732:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1732 00000100:00080000:2.0:1788876797.053742:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788876797.053761:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511267968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876797.053807:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511267456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876797.053942:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511267968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788876797.054274:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:0:1875771511267328:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876797.054285:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267712:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876797.054362:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511266688:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876797.054431:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788876797.054439:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267584:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876797.054479:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267840:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876797.070633:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267584:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876797.070656:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788876797.070659:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267840:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876797.070672:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876797.070678:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511267456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876797.070693:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511267712:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876797.694304:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876798.717814:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876799.741671:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876800.765770:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876801.790857:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876802.177123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876802.177137:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788876802.177140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876802.177144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876802.177149:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876802.177164:0:37:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1788 00000100:00080000:3.0:1788876802.177177:0:37:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788876802.177187:0:37:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876802.177255:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876802.177299:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511268096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876802.177515:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511268096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876802.177871:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511268224:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876802.184589:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511268480:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876802.185887:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1788876802.185909:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511268352:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876802.214183:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876802.214218:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876802.214226:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511268096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876802.214436:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511268352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876802.214454:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511268224:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876802.214465:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511268480:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876802.813511:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876803.837643:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876804.861765:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876805.886721:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876806.911965:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876807.293684:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876797/real 1788876797] req@ffffa061888af100 x1875771511267968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876807 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876807.293750:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511267968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876807 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876807.293774:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876807.293807:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788876807.293813:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511267968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788876807.294315:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876807.294322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788876807.294325:0:37:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876807.294332:0:37:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1788 00000100:00080000:3.0:1788876807.294345:0:37:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788876807.294356:0:37:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788876807.294436:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511268608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788876807.294466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876807.294470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876807.294472:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876807.294477:0:37:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1794 00000100:00080000:3.0:1788876807.294481:0:37:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788876807.294486:0:37:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876807.294497:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876807.294510:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511268736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876807.294654:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511268864:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876807.295027:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511269120:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876807.295068:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876807.295074:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511268992:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876807.295187:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511268608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788876807.295229:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511268736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876807.307045:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511268864:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876807.317285:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511268992:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876807.317309:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511269120:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876807.320095:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) MGC192.168.203.111@tcp: connect to target with instance 0 10000000:01000000:2.0:1788876807.320119:0:2394:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:02000400:2.0:1788876807.320130:0:2394:0:(import.c:1328:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.203.111@tcp) after server handle changed from 0xf5bf0f7e313af181 to 0xf5bf0f7e313af524 00000100:00080000:2.0:1788876807.320136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:2.0:1788876807.320142:0:2394:0:(import.c:1718:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.203.111@tcp; invalidating 00000100:00080000:2.0:1788876807.321300:0:37228:0:(import.c:1642:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.203.111@tcp to MGS@192.168.203.111@tcp 00000100:00100000:2.0:1788876807.321305:0:37228:0:(import.c:323:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:2.0:1788876807.321309:0:37228:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808003 00010000:00010000:2.0:1788876807.321325:0:37228:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886cfe00/0x7d14dcdc370488cc lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af18f expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788876807.321338:0:37228:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886cfe00/0x7d14dcdc370488cc lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af18f expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876807.321345:0:37228:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1788876807.321445:0:37228:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876807.321457:0:37228:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886cfe00/0x7d14dcdc370488cc lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af18f expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876807.321480:0:37228:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af19d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788876807.321487:0:37228:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af19d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876807.321492:0:37228:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1788876807.326930:0:37228:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876807.326957:0:37228:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06180031e00/0x7d14dcdc370488da lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af19d expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876807.327370:0:37228:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886cf800/0x7d14dcdc370488c5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af188 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788876807.327408:0:37228:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886cf800/0x7d14dcdc370488c5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af188 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876807.327417:0:37228:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1788876807.327465:0:37228:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876807.327471:0:37228:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886cf800/0x7d14dcdc370488c5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af188 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876807.327487:0:37228:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886ce000/0x7d14dcdc370488d3 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af196 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788876807.327494:0:37228:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886ce000/0x7d14dcdc370488d3 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af196 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876807.327500:0:37228:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1788876807.331352:0:37228:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876807.331377:0:37228:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886ce000/0x7d14dcdc370488d3 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af196 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:2.0:1788876807.331446:0:37228:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:2.0:1788876807.331453:0:37228:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from RECOVER to FULL 10000000:01000000:2.0:1788876807.331457:0:37228:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:2.0:1788876807.331462:0:37228:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:2.0:1788876807.331471:0:37228:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) MGC192.168.203.111@tcp: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00010000:00010000:1.0:1788876807.933628:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1788876808.381509:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1788876808.381518:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffffa06189393000 from 76 10000000:01000000:3.0:1788876808.381523:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1788876808.381723:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.381737:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.381750:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876808.381782:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269248:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788876808.381884:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f3c00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876808.393029:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269248:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876808.393066:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5e1 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.393081:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313af5e1 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876808.393185:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269376:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1788876808.393272:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f3c00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876808.398271:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269376:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00080000:3.0:1788876808.398366:0:15194:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06183c27000 already not connected (conn 126, was 0): DISCONN 00010000:00080000:3.0:1788876808.398446:0:15194:0:(ldlm_lib.c:85:import_set_conn()) imp ffffa06183c27000@lustre-MDT0000-mdc-ffffa06189393000: found existing conn 192.168.203.111@tcp, moved to head 00000100:00080000:3.0:1788876808.398453:0:15194:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876808.398459:0:15194:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 0 00000100:00080000:3.0:1788876808.398470:0:15194:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876808.398484:0:15194:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876808.398520:0:15194:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870e00 x1875771511269504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 10000000:01000000:3.0:1788876808.398588:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1788876808.398601:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5e1 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.398612:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5e1 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876808.398624:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1788876808.398628:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06189393000 from 52 10000000:01000000:3.0:1788876808.398632:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1788876808.398665:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.398680:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.398703:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876808.398726:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269632:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788876808.399256:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f3c00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876808.416152:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269632:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876808.416188:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5e8 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.416202:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313af5e8 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876808.416297:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269760:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876808.416423:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f3c00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876808.427732:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269760:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876808.427812:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269888:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1788876808.427938:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f3c00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876808.437478:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511269888:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1788876808.464909:0:37238:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 51 00000100:00100000:2.0:1788876808.473759:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270016:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876808.473806:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa061886cee00/0x7d14dcdc37048927 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5ef expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876808.473821:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa061886cee00/0x7d14dcdc37048927 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313af5ef expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876808.473933:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270144:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876808.474037:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168540 going to sleep for 11 seconds 00000100:00100000:2.0:1788876808.481710:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270144:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876808.481796:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270272:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1788876808.481922:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168540 going to sleep for 11 seconds 00000100:00100000:2.0:1788876808.502376:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270272:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1788876808.506953:0:37239:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000100:00100000:3.0:1788876808.523592:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270400:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876808.523634:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa061886cf600/0x7d14dcdc3704892e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5f6 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.523650:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa061886cf600/0x7d14dcdc3704892e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313af5f6 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876808.523762:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270528:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876808.523878:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f3d80 going to sleep for 11 seconds 00000100:00100000:3.0:1788876808.527675:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511270528:192.168.203.111@tcp:501:ll_cfg_requeue.0 10000000:01000000:3.0:1788876808.527740:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:3.0:1788876808.527752:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa061886cf600/0x7d14dcdc3704892e lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5f6 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876808.527765:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa061886cf600/0x7d14dcdc3704892e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313af5f6 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876808.957809:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876809.984044:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876811.009366:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876812.031235:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876812.413891:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511270656:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876812.413988:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511270912:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876812.414023:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1788876812.414029:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511270784:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876812.414053:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511271040:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876812.420348:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876812.420371:0:33372:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876812.420415:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876812.420417:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876812.420419:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876812.425579:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876812.425594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876812.425596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876812.425598:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876812.427197:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788876812.823618:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888ae680 x1875771511262848/t77309411330(77309411330) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876824 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.823633:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876812.823636:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 77309411329 gen 1 00000100:00100000:2.0:1788876812.823641:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888ae680 x1875771511262848/t77309411330(77309411330) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876824 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.823649:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 77309411329 last 77309411330 00000100:00080000:2.0:1788876812.823667:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888afb80 x1875771511263360/t77309411331(77309411331) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876784 ref 1 fl New:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876812.823679:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511262848:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876812.823711:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511263360:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876812.850288:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 77309411330 gen 1 00000100:00100000:2.0:1788876812.850319:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888afb80 x1875771511263360/t77309411331(77309411331) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876828 ref 2 fl Rpc:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.850334:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888afb80 x1875771511263360/t77309411331(77309411331) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876828 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.850341:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876812.850344:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 77309411330 gen 1 00000100:00100000:2.0:1788876812.850349:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888afb80 x1875771511263360/t77309411331(77309411331) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876828 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.850355:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 77309411330 last 77309411331 00000100:00080000:2.0:1788876812.850363:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888ae300 x1875771511264512/t77309411332(77309411332) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876791 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876812.850467:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511263360:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876812.850490:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511264512:192.168.203.111@tcp:36:mcreate.0 00000100:00080000:2.0:1788876812.867505:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888ae300 x1875771511264512/t77309411332(77309411332) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876828 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.867519:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876812.867522:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 77309411330 gen 1 00000100:00100000:2.0:1788876812.867528:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888afb80 x1875771511263360/t77309411331(77309411331) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876828 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.867535:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 77309411330 last 77309411332 00000100:00080000:2.0:1788876812.867540:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00010000:00080000:2.0:1788876812.870334:0:37277:0:(ldlm_request.c:2972:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffffa06189393000 to lustre-MDT0000_UUID@192.168.203.111@tcp 00010000:00010000:2.0:1788876812.870339:0:37277:0:(ldlm_request.c:2888:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffffa06189393000 (2) 00010000:00010000:2.0:1788876812.870460:0:37277:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048912 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0xf5bf0f7e313af48a expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876812.870486:0:37277:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704890b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0xf5bf0f7e313af46e expref: -99 pid: 37005 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876812.870498:0:37277:0:(ldlm_request.c:2901:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffffa06189393000 00000100:00080000:2.0:1788876812.871585:0:37277:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:2.0:1788876812.901514:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 77309411332 gen 1 00000100:00080000:2.0:1788876812.901546:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:2.0:1788876812.901559:0:2394:0:(client.c:3248:ptlrpc_resend_req()) @@@ going to resend req@ffffa061888ae680 x1875771511271168/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788876828 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.901570:0:2394:0:(client.c:3257:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa061888ae680 x1875771511271168/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788876828 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876812.901578:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00010000:00010000:2.0:1788876813.053571:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876813.447235:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.447358:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271680:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.447454:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:3.0:1788876813.447465:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511271296:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.447793:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.447824:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271808:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.458273:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511271296:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.458306:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.458321:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.467367:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271808:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876813.467433:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511271680:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876814.077708:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876815.106637:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876816.125876:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876817.150032:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876818.173732:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876818.558358:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511271936:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.558527:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272192:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.558556:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272448:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.558586:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1788876818.558591:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272064:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.558614:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272320:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.574772:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272064:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.574807:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272320:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.574818:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511271936:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.575422:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272192:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876818.575723:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511272448:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876818.736679:0:37330:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511272576:192.168.203.111@tcp:101:lfs.0 00010000:00010000:2.0:1788876818.736715:0:37330:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736730:0:37330:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736743:0:37330:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736755:0:37330:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736770:0:37330:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736783:0:37330:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736790:0:37330:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876818.736803:0:37330:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06198afed80 x1875771511272576/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876834 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876818.736819:0:37330:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736833:0:37330:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876818.736841:0:37330:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876818.736878:0:37330:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876818.736886:0:37330:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:2.0:1788876818.736893:0:37330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876818.736897:0:37330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1eefcb8 00010000:00010000:2.0:1788876818.736901:0:37330:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876818.736909:0:37330:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876818.739324:0:37330:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876818.739337:0:37330:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876818.739372:0:37330:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876818.739416:0:37330:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876818.739433:0:37330:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876818.739442:0:37330:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876818.739454:0:37330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876818.740123:0:37330:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:2.0:1788876818.740137:0:37330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1eefcb8 00010000:00010000:2.0:1788876818.740142:0:37330:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876818.740154:0:37330:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876818.740356:0:37330:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876818.740371:0:37330:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876818.741194:0:37330:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876818.741212:0:37330:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876818.741221:0:37330:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876818.741229:0:37330:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876818.741232:0:37330:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:2.0:1788876818.741239:0:37330:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1eefbb0 00010000:00010000:2.0:1788876818.741243:0:37330:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876818.741252:0:37330:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876818.741265:0:37330:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876818.741277:0:37330:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876818.742326:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.742354:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00800000:00000004:2.0:1788876818.742364:0:37330:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1760, max_age 1809 00000100:00100000:2.0:1788876818.743096:0:37330:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511272704:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876818.743209:0:37330:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061891689c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876818.751997:0:37330:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511272704:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876818.752038:0:37330:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317387/353529 objects 1023704/1024000 00000080:00200080:2.0:1788876818.752156:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.752181:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00800000:00000004:2.0:1788876818.752190:0:37330:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1760, max_age 1809 00000100:00100000:2.0:1788876818.752235:0:37330:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511272832:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876818.752342:0:37330:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061891689c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876818.772904:0:37330:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511272832:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876818.772940:0:37330:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:2.0:1788876818.773026:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773056:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773066:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773071:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773075:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773080:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773084:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773088:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773092:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773096:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773099:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773103:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773106:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773110:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773115:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773118:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773122:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773126:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773130:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773133:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773137:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773141:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773144:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773147:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773151:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773155:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773159:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773162:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773167:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773171:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773175:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773178:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773183:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773187:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773191:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773196:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773199:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773203:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773206:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773210:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773214:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773218:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773222:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773225:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773229:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773232:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773237:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773240:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773244:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773247:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773251:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773255:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773258:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773262:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773266:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773270:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773274:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773278:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773282:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773285:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773289:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773293:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773297:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773301:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773305:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773308:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773312:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773315:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773318:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:2.0:1788876818.773322:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876818.773326:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775570:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775597:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775605:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775610:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775613:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775628:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775632:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775637:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775641:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775645:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775649:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775653:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775657:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775662:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775666:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775670:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775674:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775678:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775682:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775686:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775689:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775693:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775696:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775700:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775703:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775707:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775710:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775714:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775718:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775722:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775726:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775730:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775733:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775737:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775741:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775745:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775749:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775753:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775756:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775760:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775763:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775766:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775770:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775774:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775791:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775796:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775800:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775804:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775808:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775812:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775816:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775821:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775824:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775828:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00800000:00000080:3.0:1788876818.775832:0:37330:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.775837:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.775845:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00020000:00000004:3.0:1788876818.775856:0:37330:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1760, max_age 1809 00000100:00100000:3.0:1788876818.775926:0:37330:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511272960:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876818.776115:0:37330:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203f00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876818.788098:0:37330:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511272960:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876818.788137:0:37330:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788876818.788224:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.788248:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00020000:00000004:3.0:1788876818.788256:0:37330:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1760, max_age 1809 00000100:00100000:3.0:1788876818.788298:0:37330:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511273088:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876818.788482:0:37330:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203f00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876818.792917:0:37330:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37330:1875771511273088:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876818.792955:0:37330:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788876818.793040:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793072:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793079:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793084:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793089:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793093:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793097:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793101:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793105:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793108:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793113:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793117:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793121:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793124:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793128:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793132:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793136:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793139:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793143:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793146:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793150:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793154:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793158:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793162:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793168:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793171:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793175:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793180:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793184:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793188:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793193:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793196:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793201:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793204:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793207:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793211:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793214:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793218:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793221:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793224:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793227:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793230:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793234:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793237:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793240:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793243:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793245:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793248:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793252:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793255:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793259:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793262:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793265:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793268:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793271:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793274:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793277:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793280:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793283:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793286:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793290:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793294:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793298:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793301:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793305:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793309:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793313:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793316:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793321:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793325:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793984:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.793992:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.793997:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794002:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794006:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794009:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794013:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794017:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794020:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794024:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794027:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794031:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794035:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794038:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794043:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794057:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794064:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794069:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794073:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794077:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794081:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794085:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794089:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794092:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794096:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794100:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794104:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794108:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794112:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794116:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794120:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794123:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794127:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794131:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794135:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794139:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794143:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794147:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794151:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794155:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794159:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794163:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794167:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794171:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794176:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794179:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794182:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794185:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794189:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794192:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794196:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794200:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794203:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794208:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200080:3.0:1788876818.794212:0:37330:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffeba9aa550 00020000:00000080:3.0:1788876818.794215:0:37330:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389000 uarg=0000000000000000 00000080:00200008:3.0:1788876818.794233:0:37330:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00010000:00010000:1.0:1788876819.203736:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876820.222875:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876821.245871:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876822.271438:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876823.293852:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876823.680021:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511273216:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.680135:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273472:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.680167:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273728:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.680200:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876823.680207:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273344:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.680230:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273600:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.684673:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511273216:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.684702:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273728:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.704267:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273600:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.720166:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273472:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876823.720209:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511273344:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876824.324803:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876825.347888:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876826.369422:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1788876827.634158:0:37760:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:1.0:1788876828.416895:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876828.798012:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511273856:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.798108:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511274112:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.798135:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511274368:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.798164:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1788876828.798171:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511273984:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.798194:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511274240:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.807450:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511273856:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.807882:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511273984:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.807894:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511274112:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.809748:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511274368:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876828.811964:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511274240:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876829.441221:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1788876829.794555:0:37798:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876829.794569:0:37798:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876829.794593:0:37798:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876829.794615:0:37798:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876829.794632:0:37798:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876829.794641:0:37798:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876829.794647:0:37798:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876829.794651:0:37798:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:1.0:1788876829.794658:0:37798:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac0dd3cb8 00010000:00010000:1.0:1788876829.794662:0:37798:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876829.794672:0:37798:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:1.0:1788876829.795198:0:37798:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1788876829.795203:0:37798:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876829.795211:0:37798:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876829.795220:0:37798:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876829.795229:0:37798:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1788876829.795236:0:37798:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876829.795240:0:37798:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876829.795243:0:37798:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:1.0:1788876829.795246:0:37798:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac0dd3cb8 00010000:00010000:1.0:1788876829.795249:0:37798:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876829.795256:0:37798:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876829.795346:0:37798:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876829.795355:0:37798:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876829.795362:0:37798:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876829.795372:0:37798:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:1.0:1788876829.795379:0:37798:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876829.795605:0:37798:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876829.795608:0:37798:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:1.0:1788876829.795613:0:37798:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac0dd3bb0 00010000:00010000:1.0:1788876829.795616:0:37798:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788876829.795624:0:37798:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1788876829.795634:0:37798:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:1.0:1788876829.795641:0:37798:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:1.0:1788876829.795799:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.795815:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00800000:00000004:1.0:1788876829.795822:0:37798:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1810, max_age 1820 00000100:00100000:1.0:1788876829.795870:0:37798:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274496:192.168.203.111@tcp:41:lfs.0 00000100:00100000:1.0:1788876829.795963:0:37798:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876829.798904:0:37798:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274496:192.168.203.111@tcp:41:lfs.0 00800000:00000004:1.0:1788876829.798933:0:37798:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317387/353529 objects 1023704/1024000 00000080:00200080:1.0:1788876829.798996:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.799010:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00800000:00000004:1.0:1788876829.799017:0:37798:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1810, max_age 1820 00000100:00100000:1.0:1788876829.799048:0:37798:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274624:192.168.203.111@tcp:41:lfs.0 00000100:00100000:1.0:1788876829.799105:0:37798:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876829.801592:0:37798:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274624:192.168.203.111@tcp:41:lfs.0 00800000:00000004:1.0:1788876829.801622:0:37798:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:1.0:1788876829.801685:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801701:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801710:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801715:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801720:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801724:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801728:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801731:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801735:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801738:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801742:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801745:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801750:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801753:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801758:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801761:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801764:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801767:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801770:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801774:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801780:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801784:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801788:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801792:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801796:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801800:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801804:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801807:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801811:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801815:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801819:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801822:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801826:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801829:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801833:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801837:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801841:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801844:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801847:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801850:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801854:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801857:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801860:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801863:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801867:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801871:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801875:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801878:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801882:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801886:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801889:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801893:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801897:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801900:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801903:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801907:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801910:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801913:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801917:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801920:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801924:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801927:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801931:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801934:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801937:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801940:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801944:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801947:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801951:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801954:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801958:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801962:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801966:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801969:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801973:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801977:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801980:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801982:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801986:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801989:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.801993:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.801996:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802000:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802003:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802007:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802010:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802014:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802018:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802022:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802026:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802029:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802033:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802036:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802040:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802044:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802048:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802052:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802056:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802059:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802063:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802067:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802071:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802074:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802078:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802082:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802085:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802089:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802093:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802097:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802100:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802104:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802108:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802112:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802115:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802120:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802123:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802126:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802130:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802135:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802138:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802141:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802145:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802149:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802152:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802156:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00800000:00000080:1.0:1788876829.802159:0:37798:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.802163:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.802168:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00020000:00000004:1.0:1788876829.802174:0:37798:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1810, max_age 1820 00000100:00100000:1.0:1788876829.802211:0:37798:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274752:192.168.203.111@tcp:13:lfs.0 00000100:00100000:1.0:1788876829.802286:0:37798:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876829.809137:0:37798:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274752:192.168.203.111@tcp:13:lfs.0 00020000:00000004:1.0:1788876829.809166:0:37798:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:1.0:1788876829.809235:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.809252:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00020000:00000004:1.0:1788876829.809267:0:37798:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1810, max_age 1820 00000100:00100000:1.0:1788876829.809299:0:37798:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274880:192.168.203.111@tcp:13:lfs.0 00000100:00100000:1.0:1788876829.809733:0:37798:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e600 going to sleep for 11 seconds 00000100:00100000:1.0:1788876829.818168:0:37798:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:37798:1875771511274880:192.168.203.111@tcp:13:lfs.0 00020000:00000004:1.0:1788876829.818198:0:37798:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:1.0:1788876829.818424:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818441:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818448:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818452:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818457:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818461:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818464:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818466:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818470:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818473:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818477:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818479:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818483:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818487:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818490:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818494:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818497:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818501:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818505:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818508:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818512:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818516:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818520:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818523:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818526:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818529:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818533:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818535:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818549:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818554:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818557:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818561:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818564:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818567:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818572:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818575:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818579:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818583:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818586:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818590:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818593:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818597:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818601:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818604:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818607:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818610:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818614:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818617:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818620:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818623:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818626:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818629:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818633:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818636:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818640:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818643:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818647:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818650:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818654:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818657:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818661:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818665:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818670:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818673:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818676:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818679:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818683:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818686:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818689:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818692:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818696:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818699:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818702:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818706:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818709:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818712:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818716:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818719:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818722:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818726:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818729:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818732:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818735:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818738:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818741:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818745:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818748:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818751:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818756:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818759:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818762:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818765:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818769:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818772:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818775:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818784:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818789:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818793:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818797:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818800:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818804:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818808:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818811:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818814:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818820:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818823:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818827:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818830:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818834:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818837:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818841:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818845:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818849:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818852:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818856:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818860:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818863:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818866:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818870:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818874:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818880:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818883:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818887:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818892:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200080:1.0:1788876829.818898:0:37798:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff368fb760 00020000:00000080:1.0:1788876829.818901:0:37798:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8400 uarg=0000000000000000 00000080:00200008:1.0:1788876829.818919:0:37798:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:1.0:1788876829.860323:0:37815:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876829.860368:0:37815:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876829.860406:0:37815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1788876829.860423:0:37815:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1788876829.860431:0:37815:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876829.860438:0:37815:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876829.860441:0:37815:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:1.0:1788876829.860449:0:37815:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac0ddbb58 00010000:00010000:1.0:1788876829.860453:0:37815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1788876829.860463:0:37815:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1788876829.860478:0:37815:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876829.860489:0:37815:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:1.0:1788876829.860493:0:37815:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:1.0:1788876829.860505:0:37815:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788876829.866576:0:37815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37815 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.866591:0:37815:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37815 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.866602:0:37815:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37815 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876829.866626:0:37815:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:37815:1875771511275008:192.168.203.111@tcp:101:checkstat.0 00000100:00100000:1.0:1788876829.867726:0:37815:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876829.876641:0:37815:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:37815:1875771511275008:192.168.203.111@tcp:101:checkstat.0 00010000:00010000:3.0:1788876829.876670:0:37815:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876694:0:37815:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876703:0:37815:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876715:0:37815:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876729:0:37815:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876743:0:37815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876752:0:37815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876764:0:37815:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876829.876777:0:37815:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869bb80 x1875771511275008/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/688 e 0 to 0 dl 1788876845 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876829.876791:0:37815:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876801:0:37815:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/268 (0 0) 00000080:00200000:3.0:1788876829.876808:0:37815:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788876829.876835:0:37815:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 4096 00000080:00200000:3.0:1788876829.876838:0:37815:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10c:0x0](ffffa0619828db88): rc = 0 00000080:00010000:3.0:1788876829.876843:0:37815:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc3704893c 00000080:00010000:3.0:1788876829.876853:0:37815:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00010000:3.0:1788876829.876857:0:37815:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704893c from it ffffb95ac0ddbb20 00010000:00010000:3.0:1788876829.876860:0:37815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.876870:0:37815:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876829.876909:0:37815:0:(file.c:6296:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffffa0619828db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876829.876916:0:37815:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88),name=d7.replay-single 00000002:00010000:3.0:1788876829.876937:0:37815:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876829.876947:0:37815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876829.876959:0:37815:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876829.876976:0:37815:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876829.876983:0:37815:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00010000:3.0:1788876829.876986:0:37815:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc3704893c 00000080:00010000:3.0:1788876829.876990:0:37815:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704893c from it ffffb95ac0ddbcb8 00010000:00010000:3.0:1788876829.876993:0:37815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876829.877000:0:37815:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876829.887162:0:37816:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876829.887193:0:37816:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876829.887215:0:37816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1788876829.887233:0:37816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1788876829.887241:0:37816:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876829.887415:0:37816:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876829.887419:0:37816:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:3.0:1788876829.887429:0:37816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1b53b58 00010000:00010000:3.0:1788876829.887433:0:37816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1788876829.887444:0:37816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:3.0:1788876829.887461:0:37816:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876829.887469:0:37816:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=24 00000080:00200000:3.0:1788876829.887475:0:37816:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 1 00000080:00200000:3.0:1788876829.887485:0:37816:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffffa0619828db88), flags=8 00000080:00200000:3.0:1788876829.887490:0:37816:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffffa0619828db88), intent=getattr 00000002:00010000:3.0:1788876829.887665:0:37816:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f7.replay-single,[0x0:0x0:0x0]) in obj [0x200001b71:0x10c:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876829.887719:0:37816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.887728:0:37816:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37816 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.887739:0:37816:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37816 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876829.887787:0:37816:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:37816:1875771511275136:192.168.203.111@tcp:101:checkstat.0 00000100:00100000:3.0:1788876829.887873:0:37816:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203000 going to sleep for 11 seconds 00000100:00100000:2.0:1788876829.893404:0:37816:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:37816:1875771511275136:192.168.203.111@tcp:101:checkstat.0 00010000:00010000:2.0:1788876829.893434:0:37816:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893457:0:37816:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893466:0:37816:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893480:0:37816:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893493:0:37816:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893507:0:37816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893515:0:37816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893523:0:37816:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876829.893535:0:37816:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188871180 x1875771511275136/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788876845 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876829.893565:0:37816:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893576:0:37816:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/270 (0 0) 00000080:00200000:2.0:1788876829.893583:0:37816:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200001b71:0x10c:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788876829.893605:0:37816:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10e:0x0], updating i_size 0 00000080:00200000:2.0:1788876829.893609:0:37816:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10e:0x0](ffffa0619828aa08): rc = 0 00000080:00200000:2.0:1788876829.893625:0:37816:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x10e:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1788876829.893631:0:37816:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10e:0x0](ffffa0619828aa08) for lock 0x7d14dcdc37048943 00000080:00010000:2.0:1788876829.893642:0:37816:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10e:0x0](ffffa0619828aa08) 00000080:00010000:2.0:1788876829.893645:0:37816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048943 from it ffffb95ac1b53b20 00010000:00010000:2.0:1788876829.893648:0:37816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.893659:0:37816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876829.893697:0:37816:0:(file.c:6296:ll_getattr_dentry()) START file f7.replay-single:[0x200001b71:0x10e:0x0](ffffa0619828aa08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876829.893703:0:37816:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10e:0x0](ffffa0619828aa08),name=f7.replay-single 00000002:00010000:2.0:1788876829.893724:0:37816:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10e:0x0]) in obj [0x200001b71:0x10c:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876829.893732:0:37816:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876829.893744:0:37816:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876829.893752:0:37816:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200001b71:0x10c:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876829.893757:0:37816:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10e:0x0](ffffa0619828aa08) 00000080:00010000:2.0:1788876829.893760:0:37816:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10e:0x0](ffffa0619828aa08) for lock 0x7d14dcdc37048943 00000080:00010000:2.0:1788876829.893764:0:37816:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048943 from it ffffb95ac1b53cb8 00010000:00010000:2.0:1788876829.893768:0:37816:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876829.893776:0:37816:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876829.903347:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876829.903377:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876829.903504:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1788876829.903521:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:2.0:1788876829.903528:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876829.903547:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876829.903551:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:2.0:1788876829.903558:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1527b58 00010000:00010000:2.0:1788876829.903562:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:2.0:1788876829.903570:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1788876829.903587:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876829.903596:0:37819:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=8 00000080:00200008:2.0:1788876829.903605:0:37819:0:(file.c:6296:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffffa0619828db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876829.903610:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88),name=d7.replay-single 00000002:00010000:2.0:1788876829.903623:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876829.903632:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876829.903641:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876829.903646:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876829.903651:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00010000:2.0:1788876829.903653:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc3704893c 00000080:00010000:2.0:1788876829.903657:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704893c from it ffffb95ac1527cb8 00010000:00010000:2.0:1788876829.903660:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876829.903667:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876829.903695:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876829.903703:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876829.903710:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1788876829.903718:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1788876829.903730:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876829.903735:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876829.903738:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:2.0:1788876829.903742:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1527bf8 00010000:00010000:2.0:1788876829.903745:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1788876829.903752:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1788876829.903760:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876829.903764:0:37819:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=266 00000080:00200000:2.0:1788876829.903768:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 44 00000080:00200008:2.0:1788876829.903779:0:37819:0:(file.c:1090:ll_file_open()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffffa0619828db88), flags 704400 00000002:00010000:2.0:1788876829.904320:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100240704401 00010000:00010000:2.0:1788876829.904372:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876829.904473:0:37819:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788876829.904690:0:37819:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788876829.904703:0:37819:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888adf80 x1875771511275392/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788876829.904713:0:37819:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876829.904741:0:37819:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511275392:192.168.203.111@tcp:101:rm.0 00000100:00100000:2.0:1788876829.906683:0:37819:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06191eab240 going to sleep for 11 seconds 00000100:00100000:3.0:1788876829.914109:0:37819:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511275392:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876829.914135:0:37819:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914150:0:37819:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914162:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914171:0:37819:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914180:0:37819:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914186:0:37819:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa06180030400 already unused, calling callback (ffffffffc19a4630) 00010000:00010000:3.0:1788876829.914190:0:37819:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914197:0:37819:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x206400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914204:0:37819:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x20e400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914211:0:37819:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788876829.914218:0:37819:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.914225:0:37819:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc3704894a lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876829.914243:0:37819:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa061888adf80 x1875771511275392/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788876845 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1788876829.914326:0:37819:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061888adf80 x1875771511275392/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788876845 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1788876829.914343:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876829.914368:0:37819:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 4096 00000002:00100000:3.0:1788876829.914461:0:37819:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061888adf80 x1875771511275392/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/664 e 0 to 0 dl 1788876845 ref 2 fl Complete:RPQU/604/0 rc 301/301 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1788876829.914647:0:37819:0:(file.c:6296:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffffa0619828db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876829.914654:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88),name=d7.replay-single 00000002:00010000:3.0:1788876829.914672:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876829.914688:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876829.914701:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876829.914708:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876829.914714:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00010000:3.0:1788876829.914717:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc3704893c 00000080:00010000:3.0:1788876829.914722:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704893c from it ffffb95ac1527cd8 00010000:00010000:3.0:1788876829.914726:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876829.914735:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788876829.914810:0:37819:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88) pos/size0/4096 32bit_api 0 00000002:00010000:3.0:1788876829.914825:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200001b71:0x10c:0x0], intent: readdir flags 0 00010000:00010000:3.0:1788876829.914833:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876829.914841:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000100:00100000:3.0:1788876829.918455:0:37819:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511275520:192.168.203.111@tcp:37:rm.0 00000100:00100000:3.0:1788876829.920612:0:37819:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876829.949779:0:37819:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511275776:192.168.203.111@tcp:35:rm.0 00000002:00100000:3.0:1788876829.949820:0:37819:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa061888adf80 x1875771511275392/t81604378625(81604378625) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/664 e 0 to 0 dl 1788876845 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1788876829.949898:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876829.949917:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876829.949938:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1788876829.949954:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:3.0:1788876829.949963:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876829.949970:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876829.949974:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:3.0:1788876829.949981:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1527bf8 00010000:00010000:3.0:1788876829.949986:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:3.0:1788876829.949995:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1788876829.950015:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876829.950022:0:37819:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d7.replay-single, flags=266 00000080:00200000:3.0:1788876829.950027:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 44 00000080:00200008:3.0:1788876829.950032:0:37819:0:(file.c:1090:ll_file_open()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffffa0619828db88), flags 704400 00000002:00010000:3.0:1788876829.950047:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0104240704401 00010000:00010000:3.0:1788876829.950099:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.950106:0:37819:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1788876829.950116:0:37819:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1788876829.950123:0:37819:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188870700 x1875771511276032/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1788876829.950143:0:37819:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876829.950162:0:37819:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276032:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876829.950380:0:37819:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203e40 going to sleep for 11 seconds 00000100:00100000:3.0:1788876829.957783:0:37819:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276032:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876829.957819:0:37819:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.957831:0:37819:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.957850:0:37819:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.957862:0:37819:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876829.957878:0:37819:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffffa06188870700 x1875771511276032/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788876845 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1788876829.957891:0:37819:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188870700 x1875771511276032/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788876845 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876829.957905:0:37819:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.957915:0:37819:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 16 res 8589941617/268 (0 0) 00000080:00200000:3.0:1788876829.957920:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876829.957947:0:37819:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 4096 00000080:00010000:3.0:1788876829.957952:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc37048951 00000080:00010000:3.0:1788876829.957959:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048951 from it ffffb95ac1527bd0 00010000:00010000:3.0:1788876829.957963:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.957973:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00100000:3.0:1788876829.957999:0:37819:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188870700 x1875771511276032/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/664 e 0 to 0 dl 1788876845 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1788876829.958063:0:37819:0:(file.c:6296:ll_getattr_dentry()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffffa0619828db88), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876829.958069:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88),name=d7.replay-single 00000002:00010000:3.0:1788876829.958088:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10c:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876829.958098:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876829.958109:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876829.958115:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876829.958121:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00010000:3.0:1788876829.958124:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc3704893c 00000080:00010000:3.0:1788876829.958199:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704893c from it ffffb95ac1527cd8 00010000:00010000:3.0:1788876829.958204:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876829.958270:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788876829.958340:0:37819:0:(llite_lib.c:2903:ll_statfs()) VFS Op:sb=lustre (ffffa06189393000) d7.replay-single proj=0 00000080:00000004:3.0:1788876829.958344:0:37819:0:(obd_class.h:1090:obd_statfs()) lustre-clilmv-ffffa06189393000: age 782, max_age 1820 00800000:00000004:3.0:1788876829.958354:0:37819:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1821, max_age 1820 00800000:00000004:3.0:1788876829.958357:0:37819:0:(obd_class.h:1130:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: use ffffa061893737d8 cache blocks 317387/353529 objects 1023704/1024000 00800000:00000004:3.0:1788876829.958360:0:37819:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1821, max_age 1820 00800000:00000004:3.0:1788876829.958363:0:37819:0:(obd_class.h:1130:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: use ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00000004:3.0:1788876829.958367:0:37819:0:(obd_class.h:1116:obd_statfs()) lustre-clilmv-ffffa06189393000: update ffffa06189371e18 cache blocks 634820/707058 objects 2047446/2048000 00000080:00000004:3.0:1788876829.958372:0:37819:0:(llite_lib.c:2755:ll_statfs_internal()) MDC blocks 634820/707058 objects 2047446/2048000 00000080:00000004:3.0:1788876829.958375:0:37819:0:(obd_class.h:1090:obd_statfs()) lustre-clilov-ffffa06189393000: age -290, max_age 1820 00020000:00000004:3.0:1788876829.958442:0:37819:0:(obd_class.h:1056:obd_statfs_async()) lustre-OST0000-osc-ffffa06189393000: age 1821, max_age 1820 00000008:00000004:3.0:1788876829.958447:0:37819:0:(osc_request.c:3417:osc_statfs_async()) lustre-OST0000-osc-ffffa06189393000: use ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00020000:00000004:3.0:1788876829.958452:0:37819:0:(obd_class.h:1056:obd_statfs_async()) lustre-OST0001-osc-ffffa06189393000: age 1821, max_age 1820 00000008:00000004:3.0:1788876829.958454:0:37819:0:(osc_request.c:3417:osc_statfs_async()) lustre-OST0001-osc-ffffa06189393000: use ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00000004:3.0:1788876829.958463:0:37819:0:(obd_class.h:1116:obd_statfs()) lustre-clilov-ffffa06189393000: update ffffa06189375198 cache blocks 1802724/1916558 objects 523474/524288 00000080:00000004:3.0:1788876829.958466:0:37819:0:(llite_lib.c:2766:ll_statfs_internal()) OSC blocks 1802724/1916558 objects 523474/524288 00000080:00200000:3.0:1788876829.958485:0:37819:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88) pos/size0/4096 32bit_api 0 00000080:00200000:3.0:1788876829.958508:0:37819:0:(dir.c:332:ll_iterate()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88) pos/size18446744073709551614/4096 32bit_api 0 00000080:00200000:3.0:1788876829.958530:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 1 00000080:00200000:3.0:1788876829.958549:0:37819:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f7.replay-single, flags=0 00000080:00200000:3.0:1788876829.958556:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x10c:0x0](ffffa0619828db88), inode mode 41ed mask 3 00000080:00200000:3.0:1788876829.958561:0:37819:0:(namei.c:2351:ll_unlink()) VFS Op:name=f7.replay-single, dir=[0x200001b71:0x10c:0x0](ffffa0619828db88) 00010000:00010000:3.0:1788876829.958586:0:37819:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876829.958603:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876829.958669:0:37819:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876829.958681:0:37819:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x10e:0x0]: losing layout lock 00010000:00010000:3.0:1788876829.958716:0:37819:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876829.958724:0:37819:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876829.958731:0:37819:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876829.958734:0:37819:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048943 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10e:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313af90d expref: -99 pid: 37816 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876829.958750:0:37819:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e800/0x7d14dcdc3704893c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af8f1 expref: -99 pid: 37815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:3.0:1788876829.958766:0:37819:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876829.958774:0:37819:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188870a80 x1875771511276288/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876829.958795:0:37819:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276288:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876829.958910:0:37819:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203e40 going to sleep for 11 seconds 00000100:00100000:3.0:1788876829.972605:0:37819:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276288:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876829.972660:0:37819:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x10e:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788876829.972670:0:37819:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x10e:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876829.972677:0:37819:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10e:0x0](ffffa0619828aa08) 00000080:00200008:3.0:1788876829.972727:0:37819:0:(file.c:479:ll_file_release()) START file d7.replay-single:[0x200001b71:0x10c:0x0](ffffa0619828db88), flags 704000 00010000:00010000:3.0:1788876829.972754:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876829.972791:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876829.972806:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876829.972816:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1788876829.972831:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:3.0:1788876829.972838:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876829.972844:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876829.972848:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:3.0:1788876829.972854:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1527c10 00010000:00010000:3.0:1788876829.972858:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:3.0:1788876829.972865:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:3.0:1788876829.972881:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876829.972895:0:37819:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=0 00000080:00200000:3.0:1788876829.972900:0:37819:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876829.972911:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: d7.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876829.972953:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.972960:0:37819:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876829.972970:0:37819:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 37819 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876829.972989:0:37819:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276416:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876829.973060:0:37819:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203e40 going to sleep for 11 seconds 00000100:00100000:1.0:1788876829.980090:0:37819:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276416:192.168.203.111@tcp:101:rm.0 00010000:00010000:1.0:1788876829.980125:0:37819:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980148:0:37819:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980156:0:37819:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980169:0:37819:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980180:0:37819:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980193:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980201:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980208:0:37819:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876829.980220:0:37819:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188872d80 x1875771511276416/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/688 e 0 to 0 dl 1788876845 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876829.980235:0:37819:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980247:0:37819:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/268 (0 0) 00000080:00200000:1.0:1788876829.980253:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788876829.980286:0:37819:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10c:0x0], updating i_size 4096 00000080:00200000:1.0:1788876829.980289:0:37819:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10c:0x0](ffffa0619828db88): rc = 0 00000080:00010000:1.0:1788876829.980296:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) for lock 0x7d14dcdc37048958 00000080:00010000:1.0:1788876829.980307:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10c:0x0](ffffa0619828db88) 00000080:00010000:1.0:1788876829.980311:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048958 from it ffffb95ac1527e00 00010000:00010000:1.0:1788876829.980315:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876829.980324:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876829.980362:0:37819:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876829.980429:0:37819:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876829.980441:0:37819:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1788876829.980452:0:37819:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:1.0:1788876829.980458:0:37819:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876829.980464:0:37819:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876829.980467:0:37819:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048935 00000080:00010000:1.0:1788876829.980471:0:37819:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048935 from it ffffb95ac1527d88 00010000:00010000:1.0:1788876829.980474:0:37819:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:1.0:1788876829.980482:0:37819:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000080:00200000:1.0:1788876829.980492:0:37819:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876829.980499:0:37819:0:(namei.c:2261:ll_rmdir()) VFS Op:name=d7.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00000100:00100000:1.0:1788876829.985974:0:37819:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276672:192.168.203.111@tcp:35:rm.0 00000002:00100000:1.0:1788876829.986016:0:37819:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06188870700 x1875771511276032/t81604378627(81604378627) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/664 e 0 to 0 dl 1788876845 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'rm.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1788876829.986104:0:37819:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876829.986115:0:37819:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 1/0,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876829.986123:0:37819:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00010000:00010000:1.0:1788876829.986129:0:37819:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:1.0:1788876829.986132:0:37819:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048958 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10c:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af968 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876829.986142:0:37819:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048951 lrc: 0/0,0 mode: --/CR res: [0x200001b71:0x10c:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af929 expref: -99 pid: 37819 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876829.986162:0:37819:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048935 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af8d5 expref: -99 pid: 37330 timeout: 0 lvb_type: 0 lru_score: 13 lru_type: 1 00000020:00100000:1.0:1788876829.986184:0:37819:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876829.986191:0:37819:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188699180 x1875771511276928/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876829.986213:0:37819:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:37819:1875771511276928:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876829.986290:0:37819:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121ef00 going to sleep for 11 seconds 00010000:00010000:1.0:1788876830.486756:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876831.488628:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876832.516603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876833.538475:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876833.919397:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511277056:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.919496:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277312:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.920030:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277568:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.920141:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788876833.920150:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277184:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.920177:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277440:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.926876:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277184:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.926898:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277440:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.926909:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511277056:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.926922:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277312:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876833.926933:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511277568:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876834.557645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876835.584603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876836.608358:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876837.629669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1788876837.867978:0:38411:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876837.867995:0:38411:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876837.868141:0:38411:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876837.868216:0:38411:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38411 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876837.868228:0:38411:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38411 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876837.868239:0:38411:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38411 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876837.868270:0:38411:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511277696:192.168.203.111@tcp:101:lfs.0 00000100:00100000:1.0:1788876837.871462:0:38411:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187aae600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876837.890767:0:38411:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 81604378630 gen 1 00000100:00100000:3.0:1788876837.890832:0:38411:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511277696:192.168.203.111@tcp:101:lfs.0 00010000:00010000:3.0:1788876837.890864:0:38411:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876837.890880:0:38411:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876837.890893:0:38411:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876837.890907:0:38411:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876837.890922:0:38411:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876837.890931:0:38411:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876837.890938:0:38411:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876837.890951:0:38411:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188699f80 x1875771511277696/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876853 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876837.890981:0:38411:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876837.890999:0:38411:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876837.891015:0:38411:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876837.891436:0:38411:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876837.891447:0:38411:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704895f 00000080:00010000:2.0:1788876837.891454:0:38411:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876837.891458:0:38411:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704895f from it ffffb95ac156fcb8 00010000:00010000:2.0:1788876837.891462:0:38411:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876837.891475:0:38411:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876837.896055:0:38411:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876837.896073:0:38411:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876837.896108:0:38411:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876837.896131:0:38411:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876837.896151:0:38411:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876837.896164:0:38411:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876837.896174:0:38411:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876837.896178:0:38411:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704895f 00000080:00010000:2.0:1788876837.896188:0:38411:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704895f from it ffffb95ac156fcb8 00010000:00010000:2.0:1788876837.896194:0:38411:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876837.896205:0:38411:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876837.896536:0:38411:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876837.896546:0:38411:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876837.896554:0:38411:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876837.896565:0:38411:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876837.896572:0:38411:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876837.896582:0:38411:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876837.896585:0:38411:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704895f 00000080:00010000:2.0:1788876837.896589:0:38411:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704895f from it ffffb95ac156fbb0 00010000:00010000:2.0:1788876837.896593:0:38411:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876837.896600:0:38411:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876837.896608:0:38411:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876837.896618:0:38411:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876837.896724:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.896743:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00800000:00000004:2.0:1788876837.896752:0:38411:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1821, max_age 1828 00000100:00100000:2.0:1788876837.896808:0:38411:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511277824:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876837.896922:0:38411:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876837.904736:0:38411:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511277824:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876837.904774:0:38411:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023706/1024000 00000080:00200080:2.0:1788876837.904877:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.904901:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00800000:00000004:2.0:1788876837.904911:0:38411:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1821, max_age 1828 00000100:00100000:2.0:1788876837.904953:0:38411:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511277952:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876837.905190:0:38411:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876837.928497:0:38411:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511277952:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876837.928535:0:38411:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:2.0:1788876837.928623:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928645:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928654:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928659:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928664:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928667:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928671:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928675:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928678:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928682:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928686:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928689:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928694:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928698:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928703:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928705:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928709:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928713:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928717:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928720:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928724:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928728:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928732:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928735:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928740:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928744:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928748:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928752:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928756:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928760:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928764:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928769:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928773:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928777:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928782:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928786:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928790:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928795:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928799:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928803:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928808:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928812:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928816:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928820:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928824:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928828:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928832:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928836:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928840:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928844:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928848:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928852:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928855:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928859:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928863:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928868:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928872:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928875:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928879:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928883:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928887:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928891:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928896:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928900:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928905:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928909:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928913:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928916:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928921:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928925:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928929:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928933:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928937:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928941:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928946:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928949:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928954:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928957:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928962:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928966:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928970:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928974:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928978:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928981:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928985:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928989:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.928994:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.928999:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929003:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929006:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929010:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929014:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929018:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929021:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929025:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929028:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929032:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929035:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929039:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929042:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929046:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929050:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929053:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929057:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929061:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929064:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929068:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929071:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929075:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929079:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929084:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929087:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929092:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929096:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929100:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929104:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929109:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929112:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929117:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929121:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929125:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929129:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929133:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929137:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929141:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00800000:00000080:2.0:1788876837.929145:0:38411:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.929150:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.929158:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00020000:00000004:2.0:1788876837.929168:0:38411:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1821, max_age 1828 00000100:00100000:2.0:1788876837.929219:0:38411:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511278080:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876837.929314:0:38411:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876837.939907:0:38411:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511278080:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876837.939945:0:38411:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876837.940031:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.940053:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00020000:00000004:2.0:1788876837.940061:0:38411:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1821, max_age 1828 00000100:00100000:2.0:1788876837.940102:0:38411:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511278208:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876837.940235:0:38411:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876837.954803:0:38411:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38411:1875771511278208:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876837.954841:0:38411:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788876837.954931:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.954956:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.954964:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.954968:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.954972:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.954976:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.954980:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.954983:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.954988:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.954992:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.954996:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.954999:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955003:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955007:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955011:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955015:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955018:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955021:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955026:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955029:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955035:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955038:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955042:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955045:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955049:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955052:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955055:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955058:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955062:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955066:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955070:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955073:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955078:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955082:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955086:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955089:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955094:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955097:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955101:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955105:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955109:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955113:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955117:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955121:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955125:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955129:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955133:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955137:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955141:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955145:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955149:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955155:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955158:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955161:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955165:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955169:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955172:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955177:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955181:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955184:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955188:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955201:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955207:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955212:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955216:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955220:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955224:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955227:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955231:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955235:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955239:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955242:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955246:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955249:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955253:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955256:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955260:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955263:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955268:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955272:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955275:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955278:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955282:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955285:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955289:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955291:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955295:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955298:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955302:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955306:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955309:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955312:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955315:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955329:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955334:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955339:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955343:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955346:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955350:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955355:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955358:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955362:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955366:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955369:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955373:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955377:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955432:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955438:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955444:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955448:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955453:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955457:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955460:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955464:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955467:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955471:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955476:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955489:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955494:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955499:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955504:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955508:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955512:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955516:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200080:2.0:1788876837.955521:0:38411:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff619cfa40 00020000:00000080:2.0:1788876837.955525:0:38411:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e05c00 uarg=0000000000000000 00000080:00200008:2.0:1788876837.955544:0:38411:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000002:00010000:2.0:1788876838.006669:0:38418:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876838.006695:0:38418:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876838.006712:0:38418:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876838.006721:0:38418:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876838.006731:0:38418:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876838.006736:0:38418:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704895f 00000080:00010000:2.0:1788876838.006744:0:38418:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704895f from it ffffb95ac1d27bd0 00010000:00010000:2.0:1788876838.006748:0:38418:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876838.006757:0:38418:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876838.006776:0:38418:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876838.006798:0:38418:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:2.0:1788876838.006802:0:38418:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876838.006810:0:38418:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876838.006816:0:38418:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876838.006825:0:38418:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876838.006832:0:38418:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876838.006836:0:38418:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876838.006838:0:38418:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704895f 00000080:00010000:2.0:1788876838.006842:0:38418:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704895f from it ffffb95ac1d27cd0 00010000:00010000:2.0:1788876838.006845:0:38418:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876838.006853:0:38418:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876838.006862:0:38418:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876838.006869:0:38418:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876838.006875:0:38418:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876838.006881:0:38418:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876838.006889:0:38418:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876838.006896:0:38418:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876838.006899:0:38418:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876838.006902:0:38418:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704895f 00000080:00010000:2.0:1788876838.006906:0:38418:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704895f from it ffffb95ac1d27db0 00010000:00010000:2.0:1788876838.006909:0:38418:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876838.006915:0:38418:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1788876838.006924:0:38418:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876838.006930:0:38418:0:(namei.c:2062:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:2.0:1788876838.006934:0:38418:0:(namei.c:2017:ll_mknod()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:2.0:1788876838.006969:0:38418:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876838.007030:0:38418:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876838.007039:0:38418:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876838.007042:0:38418:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ea00/0x7d14dcdc3704895f lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af98b expref: -99 pid: 38411 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:2.0:1788876838.007073:0:38418:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876838.007091:0:38418:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888aca80 x1875771511278464/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876838.007121:0:38418:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:38418:1875771511278464:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876838.007672:0:38418:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:3.0:1788876838.046030:0:38421:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:38421:1875771511278592:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876838.046068:0:38421:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046084:0:38421:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046100:0:38421:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046112:0:38421:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046127:0:38421:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046136:0:38421:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046144:0:38421:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876838.046160:0:38421:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06198afd180 x1875771511278592/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876854 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876838.046185:0:38421:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046213:0:38421:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876838.046222:0:38421:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876838.046266:0:38421:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876838.046276:0:38421:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048966 00000080:00010000:3.0:1788876838.046284:0:38421:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876838.046289:0:38421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048966 from it ffffb95ac156fb58 00010000:00010000:3.0:1788876838.046293:0:38421:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046302:0:38421:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876838.046342:0:38421:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876838.046371:0:38421:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:3.0:1788876838.046376:0:38421:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876838.046441:0:38421:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: fsa-oleg311-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876838.046504:0:38421:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 38421 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046513:0:38421:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38421 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.046524:0:38421:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38421 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876838.046552:0:38421:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:38421:1875771511278720:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876838.046658:0:38421:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187ae1300 going to sleep for 11 seconds 00000100:00100000:3.0:1788876838.050271:0:38421:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:38421:1875771511278720:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876838.050312:0:38421:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050352:0:38421:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050361:0:38421:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050373:0:38421:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050582:0:38421:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050709:0:38421:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050719:0:38421:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050726:0:38421:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876838.050747:0:38421:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188870a80 x1875771511278720/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 600/584 e 0 to 0 dl 1788876854 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876838.050762:0:38421:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050775:0:38421:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/271 (0 0) 00000080:00200000:3.0:1788876838.050783:0:38421:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788876838.050812:0:38421:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x10f:0x0], updating i_size 0 00000080:00200000:3.0:1788876838.050817:0:38421:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x10f:0x0](ffffa061981b9148): rc = 0 00000080:00200000:3.0:1788876838.050840:0:38421:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x10f:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1788876838.050848:0:38421:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10f:0x0](ffffa061981b9148) for lock 0x7d14dcdc3704896d 00000080:00010000:3.0:1788876838.050866:0:38421:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10f:0x0](ffffa061981b9148) 00000080:00010000:3.0:1788876838.050870:0:38421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704896d from it ffffb95ac156fb20 00010000:00010000:3.0:1788876838.050874:0:38421:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876838.050882:0:38421:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876838.050931:0:38421:0:(file.c:6296:ll_getattr_dentry()) START file fsa-oleg311-client.virtnet:[0x200001b71:0x10f:0x0](ffffa061981b9148), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876838.050941:0:38421:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x10f:0x0](ffffa061981b9148),name=fsa-oleg311-client.virtnet 00000002:00010000:3.0:1788876838.050971:0:38421:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x10f:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876838.050982:0:38421:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876838.051000:0:38421:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876838.051008:0:38421:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876838.051013:0:38421:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x10f:0x0](ffffa061981b9148) 00000080:00010000:3.0:1788876838.051016:0:38421:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x10f:0x0](ffffa061981b9148) for lock 0x7d14dcdc3704896d 00000080:00010000:3.0:1788876838.051020:0:38421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704896d from it ffffb95ac156fcb8 00010000:00010000:3.0:1788876838.051023:0:38421:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876838.051033:0:38421:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876838.051110:0:38421:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876838.051120:0:38421:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876838.051129:0:38421:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876838.051138:0:38421:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876838.051145:0:38421:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876838.051149:0:38421:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876838.051152:0:38421:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048966 00000080:00010000:3.0:1788876838.051155:0:38421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048966 from it ffffb95ac156fc10 00010000:00010000:3.0:1788876838.051158:0:38421:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876838.051166:0:38421:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876838.051175:0:38421:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876838.051187:0:38421:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg311-client.virtnet, flags=0 00000080:00200000:3.0:1788876838.051207:0:38421:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876838.051214:0:38421:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876838.051220:0:38421:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876838.051230:0:38421:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876838.051237:0:38421:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876838.051241:0:38421:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876838.051243:0:38421:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048966 00000080:00010000:3.0:1788876838.051247:0:38421:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048966 from it ffffb95ac156fd68 00010000:00010000:3.0:1788876838.051250:0:38421:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876838.051258:0:38421:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876838.051267:0:38421:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876838.051274:0:38421:0:(namei.c:2351:ll_unlink()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:3.0:1788876838.051302:0:38421:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876838.051343:0:38421:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876838.051355:0:38421:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x10f:0x0]: losing layout lock 00010000:00010000:3.0:1788876838.051670:0:38421:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876838.051679:0:38421:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876838.051686:0:38421:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1788876838.051689:0:38421:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc3704896d lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x10f:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313af9c3 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876838.051710:0:38421:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048966 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af9a7 expref: -99 pid: 38421 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:3.0:1788876838.051734:0:38421:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876838.051745:0:38421:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188870a80 x1875771511278976/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876838.051777:0:38421:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:38421:1875771511278976:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876838.051893:0:38421:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187ae1300 going to sleep for 11 seconds 00000100:00100000:3.0:1788876838.056001:0:38421:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:38421:1875771511278976:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876838.056095:0:38421:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x10f:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788876838.056110:0:38421:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x10f:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876838.056125:0:38421:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x10f:0x0](ffffa061981b9148) 00010000:00010000:1.0:1788876838.656340:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876839.679417:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876840.701701:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876841.726881:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876842.750845:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876843.773590:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876844.159130:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511279744:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.159221:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511280000:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.159250:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511280256:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.159292:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788876844.159299:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511279872:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.159323:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511280128:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.185803:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511279872:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.185823:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511280128:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.185833:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511279744:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.185842:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511280000:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876844.185852:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511280256:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876844.797941:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876844.990920:0:38487:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:2.0:1788876845.148208:0:38489:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876845.148245:0:38489:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876845.148325:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.148336:0:38489:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.148347:0:38489:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876845.148372:0:38489:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511280384:192.168.203.111@tcp:101:multiop.0 00000100:00100000:2.0:1788876845.148516:0:38489:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10300 going to sleep for 11 seconds 00000100:00100000:2.0:1788876845.154925:0:38489:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511280384:192.168.203.111@tcp:101:multiop.0 00010000:00010000:2.0:1788876845.154959:0:38489:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.154974:0:38489:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.154988:0:38489:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.155001:0:38489:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.155015:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.155028:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.155036:0:38489:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876845.155046:0:38489:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888ac380 x1875771511280384/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876845.155061:0:38489:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.155074:0:38489:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876845.155080:0:38489:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876845.155105:0:38489:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876845.155111:0:38489:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048974 00000080:00010000:2.0:1788876845.155116:0:38489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876845.155119:0:38489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048974 from it ffffb95ac1cf7bd0 00010000:00010000:2.0:1788876845.155122:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.155130:0:38489:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876845.155155:0:38489:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876845.155176:0:38489:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:2.0:1788876845.155181:0:38489:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876845.155196:0:38489:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876845.155205:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876845.155217:0:38489:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876845.155224:0:38489:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876845.155228:0:38489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876845.155231:0:38489:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048974 00000080:00010000:2.0:1788876845.155235:0:38489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048974 from it ffffb95ac1cf7cd0 00010000:00010000:2.0:1788876845.155238:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876845.155245:0:38489:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876845.155254:0:38489:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876845.155260:0:38489:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876845.155267:0:38489:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876845.155273:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876845.155280:0:38489:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876845.155286:0:38489:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876845.155290:0:38489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876845.155292:0:38489:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048974 00000080:00010000:2.0:1788876845.155296:0:38489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048974 from it ffffb95ac1cf7db0 00010000:00010000:2.0:1788876845.155299:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876845.155306:0:38489:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876845.155313:0:38489:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876845.155318:0:38489:0:(namei.c:2062:ll_create_nd()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:2.0:1788876845.155322:0:38489:0:(namei.c:2017:ll_mknod()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:2.0:1788876845.155346:0:38489:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876845.155783:0:38489:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876845.155793:0:38489:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876845.155796:0:38489:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc37048974 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313af9ed expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1788876845.155822:0:38489:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876845.155829:0:38489:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ac380 x1875771511280640/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876845.155905:0:38489:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511280640:192.168.203.111@tcp:36:multiop.0 00000100:00100000:2.0:1788876845.156035:0:38489:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10300 going to sleep for 11 seconds 00000100:00100000:2.0:1788876845.170786:0:38489:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511280640:192.168.203.111@tcp:36:multiop.0 00000080:00200000:2.0:1788876845.170835:0:38489:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x110:0x0](ffffa06198289148) 00000080:00200000:2.0:1788876845.170933:0:38489:0:(namei.c:2070:ll_create_nd()) VFS Op:name=f8.replay-single, unhashed 1 00000080:00200000:2.0:1788876845.171103:0:38489:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876845.171121:0:38489:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876845.171329:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.171340:0:38489:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.171350:0:38489:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876845.171369:0:38489:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511280768:192.168.203.111@tcp:101:multiop.0 00000100:00100000:2.0:1788876845.171652:0:38489:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10300 going to sleep for 11 seconds 00000100:00100000:2.0:1788876845.174436:0:38489:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511280768:192.168.203.111@tcp:101:multiop.0 00010000:00010000:2.0:1788876845.174468:0:38489:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174482:0:38489:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174496:0:38489:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174508:0:38489:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174522:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174537:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174545:0:38489:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876845.174558:0:38489:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888ad880 x1875771511280768/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876861 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876845.174572:0:38489:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174588:0:38489:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876845.174595:0:38489:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876845.174627:0:38489:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876845.174634:0:38489:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc3704897b 00000080:00010000:2.0:1788876845.174639:0:38489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876845.174642:0:38489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc3704897b from it ffffb95ac1cf7bf8 00010000:00010000:2.0:1788876845.174646:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174656:0:38489:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030600/0x7d14dcdc3704897b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa09 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876845.174685:0:38489:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876845.174713:0:38489:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa061839fb500, open_flags 8000, mode 0 opened 0 00000080:00200000:2.0:1788876845.174720:0:38489:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:2.0:1788876845.174743:0:38489:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f8.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100001 00010000:00010000:2.0:1788876845.174793:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876845.174800:0:38489:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788876845.174811:0:38489:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788876845.174817:0:38489:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ad880 x1875771511281024/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788876845.174825:0:38489:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 38489 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876845.174845:0:38489:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511281024:192.168.203.111@tcp:101:multiop.0 00000100:00100000:3.0:1788876845.175375:0:38489:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10300 going to sleep for 11 seconds 00000100:00100000:3.0:1788876845.182628:0:38489:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511281024:192.168.203.111@tcp:101:multiop.0 00010000:00010000:3.0:1788876845.182658:0:38489:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182680:0:38489:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182688:0:38489:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182700:0:38489:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182711:0:38489:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182725:0:38489:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182732:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182739:0:38489:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876845.182751:0:38489:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x800002b, status=0 req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1788876845.182769:0:38489:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876845.182781:0:38489:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.182792:0:38489:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/272 (0 0) 00000080:00200000:3.0:1788876845.182798:0:38489:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788876845.182819:0:38489:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x110:0x0], updating i_size 0 00000080:00200000:3.0:1788876845.182822:0:38489:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x110:0x0](ffffa06198289148): rc = 0 00000080:00200000:3.0:1788876845.182836:0:38489:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x110:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1788876845.182842:0:38489:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) for lock 0x7d14dcdc37048982 00000080:00010000:3.0:1788876845.182906:0:38489:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) 00000080:00200008:3.0:1788876845.182922:0:38489:0:(file.c:1090:ll_file_open()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffffa06198289148), flags 100000 00000002:00100000:3.0:1788876845.182945:0:38489:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876861 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1788876845.182986:0:38489:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048982 from it ffffa061888c0060 00010000:00010000:3.0:1788876845.182991:0:38489:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876845.183000:0:38489:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048982 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afa25 expref: -99 pid: 38489 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876845.183022:0:38489:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x110:0x0](ffffa06198289148), inode mode 81a4 mask 44 00010000:00010000:2.0:1788876845.821754:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876846.845738:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876847.869637:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876848.894736:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876849.290824:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511281408:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788876849.290964:0:2396:0:(recover.c:205:ptlrpc_request_handle_notconn()) import lustre-MDT0000-mdc-ffffa06189393000 of lustre-MDT0000_UUID@192.168.203.111@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1788876849.290972:0:2396:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1788876849.290976:0:2396:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa06189393000: Connection to lustre-MDT0000 (at 192.168.203.111@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:1.0:1788876849.290984:0:2396:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788876849.290990:0:2396:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1800 00000100:00080000:1.0:1788876849.291001:0:2396:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788876849.291014:0:2396:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788876849.291065:0:2396:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699500 x1875771511281792/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876849.291096:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511281280:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876849.300917:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511281664:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876849.300938:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511281536:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876849.301306:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876849.301313:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876849.301317:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511281792:192.168.203.111@tcp:38:ptlrpcd_00_01.0 00010000:00010000:1.0:1788876849.920181:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876850.944984:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876851.970700:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876852.993407:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876854.015795:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876854.398409:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876854.398416:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876854.398421:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788876854.398424:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511281920:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876854.398431:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1841 00000100:00080000:3.0:1788876854.398443:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788876854.398451:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876854.398463:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876854.398483:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511282048/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876854.398547:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511282304:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876854.398582:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876854.398590:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511282176:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876854.398617:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511282432:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876854.411771:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876854.411784:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876854.411790:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511282048:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876854.413500:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511282176:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876854.413517:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511282432:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876854.413528:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511282304:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876855.041617:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876856.061621:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876857.085627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876858.112499:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876859.133820:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788876859.518485:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788876859.518492:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788876859.518496:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788876859.518508:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1846 00000100:00080000:1.0:1788876859.518519:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1788876859.518525:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788876859.518542:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788876859.518560:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511282688/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876859.518653:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511282688:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788876859.529035:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511282944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876859.534031:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511282816:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876859.534057:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511283072:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876859.534584:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876859.534606:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876859.538713:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511282688:192.168.203.111@tcp:38:kworker.0 00010000:00010000:1.0:1788876860.157676:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876861.181639:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876862.205527:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876863.231756:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876864.253570:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788876864.637912:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876864.637920:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788876864.637925:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876864.637937:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1851 00000100:00080000:2.0:1788876864.637948:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1788876864.637956:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876864.637967:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876864.637987:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511283328/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876864.638097:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511283200:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876864.638183:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511283584:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876864.638223:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876864.638232:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511283456:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876864.638260:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511283712:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788876864.665598:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788876864.665613:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788876864.665618:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511283328:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876864.668877:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511283456:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876864.668897:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511283712:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876864.668909:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511283584:192.168.203.111@tcp:400:kworker.0 00000100:00000400:1.0:1788876865.469537:0:2396:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876849/real 1788876849] req@ffffa0618869b100 x1875771511281152/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876865 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788876865.469570:0:2396:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa0618869b100 x1875771511281152/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788876865.469580:0:2396:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1788876865.469584:0:2396:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.111@tcp: Connection to MGS (at 192.168.203.111@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1788876865.469588:0:2396:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.111@tcp for MGC192.168.203.111@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1788876865.469591:0:2396:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1788876865.469595:0:2396:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa0618869b100 x1875771511281152/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788876865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788876865.469606:0:2396:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188872680 x1875771511281920/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876870 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788876865.469631:0:2396:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188698000 x1875771511282560/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876875 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788876865.469652:0:2396:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888ac000 x1875771511283200/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788876880 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1788876865.469661:0:2396:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1788876865.469665:0:2396:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1788876865.469667:0:2396:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1788876865.469669:0:2396:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:1.0:1788876865.469699:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511281152:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876865.469784:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876865.469789:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788876865.469792:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876865.469798:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1799 00000100:00080000:3.0:1788876865.469810:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788876865.469840:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511283840/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788876865.469864:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876865.469867:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876865.469869:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876865.469873:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1856 00000100:00080000:3.0:1788876865.469877:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788876865.469881:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876865.469891:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876865.469902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511283968/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876865.469951:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511284224:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876865.470029:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788876865.470035:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511284096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876865.470059:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511284352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876865.470238:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511283840:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788876865.470272:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511283968:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876865.470975:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511283200:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876865.470986:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511281920:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788876865.483912:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788876865.483924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788876865.483930:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511283968:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876865.485069:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511284096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876865.485501:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511284352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876865.485514:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511284224:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876866.301720:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876867.328038:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876869.374703:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876870.400267:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788876870.783720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876870.783734:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788876870.783738:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788876870.783742:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788876870.783747:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876870.783761:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1857 00000100:00080000:2.0:1788876870.783776:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788876870.783785:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876870.783802:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876870.783849:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511284480/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876870.783955:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511284736:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876870.784071:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876870.784078:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511284608:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876870.784116:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511284864:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876870.784418:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511284480:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788876870.791331:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788876870.791350:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788876870.791358:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511284480:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876870.792102:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511284608:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876870.802427:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511284864:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876870.802468:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511284736:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876872.447563:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876873.469743:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876875.519895:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876875.902183:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876865/real 1788876865] req@ffffa06188871880 x1875771511283840/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876875 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876875.902205:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511283840/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876875 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876875.902228:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511284992:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788876875.902279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876875.902288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788876875.902293:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511283840:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788876875.903646:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511285504:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788876875.926973:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511285120:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788876875.932057:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) MGC192.168.203.111@tcp: connect to target with instance 0 10000000:01000000:2.0:1788876875.932073:0:2394:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:02000400:2.0:1788876875.932081:0:2394:0:(import.c:1328:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.203.111@tcp) after server handle changed from 0xf5bf0f7e313af524 to 0xf5bf0f7e313afb67 00000100:00080000:2.0:1788876875.932087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:2.0:1788876875.932092:0:2394:0:(import.c:1718:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.203.111@tcp; invalidating 00000100:00100000:1.0:1788876875.932238:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511285376:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876875.932266:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511285248:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876875.932436:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511285504:192.168.203.111@tcp:250:kworker.0 00010000:00080000:2.0:1788876875.932456:0:2394:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=0) req@ffffa06198afc700 x1875771511284992/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/416 e 0 to 0 dl 1788876900 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876875.932468:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa06189393000: connect to target with instance 20 00000100:00080000:2.0:1788876875.932482:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:2.0:1788876875.932488:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876875.932490:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 81604378630 gen 1 00000100:00100000:2.0:1788876875.932497:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888aca80 x1875771511278464/t81604378631(81604378631) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876854 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876875.932505:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 81604378630 last 0 00000100:00080000:2.0:1788876875.932512:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888aca80 x1875771511278464/t81604378631(81604378631) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876854 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876875.932522:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511284992:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876875.932543:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511278464:192.168.203.111@tcp:36:mcreate.0 00000100:00080000:1.0:1788876875.934698:0:38722:0:(import.c:1642:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.203.111@tcp to MGS@192.168.203.111@tcp 00000100:00100000:1.0:1788876875.934705:0:38722:0:(import.c:323:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:1.0:1788876875.934709:0:38722:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808003 00010000:00010000:1.0:1788876875.934726:0:38722:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af5e8 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876875.934738:0:38722:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af5e8 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876875.934744:0:38722:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1788876875.934767:0:38722:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876875.934776:0:38722:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887b800/0x7d14dcdc37048920 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af5e8 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876875.934792:0:38722:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886cf600/0x7d14dcdc3704892e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af5f6 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876875.934797:0:38722:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886cf600/0x7d14dcdc3704892e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af5f6 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876875.934802:0:38722:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1788876875.934805:0:38722:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876875.934808:0:38722:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886cf600/0x7d14dcdc3704892e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af5f6 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876875.934821:0:38722:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af5e1 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876875.934827:0:38722:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af5e1 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876875.934832:0:38722:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:1.0:1788876875.934835:0:38722:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876875.934839:0:38722:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048919 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af5e1 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876875.934851:0:38722:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886cee00/0x7d14dcdc37048927 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313af5ef expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1788876875.934856:0:38722:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886cee00/0x7d14dcdc37048927 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313af5ef expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788876875.934861:0:38722:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:1.0:1788876875.934863:0:38722:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788876875.934866:0:38722:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886cee00/0x7d14dcdc37048927 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313af5ef expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1788876875.934880:0:38722:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:1.0:1788876875.934884:0:38722:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from RECOVER to FULL 10000000:01000000:1.0:1788876875.934886:0:38722:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:1.0:1788876875.934889:0:38722:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:1.0:1788876875.934893:0:38722:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) MGC192.168.203.111@tcp: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00010000:00010000:1.0:1788876876.541680:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1788876877.823417:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1788876877.823425:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffffa06189393000 from 80 10000000:01000000:3.0:1788876877.823430:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1788876877.823476:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876877.823489:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876877.823501:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876877.823528:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511285632:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788876877.823626:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e900 going to sleep for 11 seconds 00000100:00100000:3.0:1788876877.835459:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511285632:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876877.835511:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc32 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876877.835527:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313afc32 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876877.835632:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511285760:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1788876877.835748:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e900 going to sleep for 11 seconds 00000100:00100000:3.0:1788876877.852901:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511285760:192.168.203.111@tcp:256:ll_cfg_requeue.0 10000000:01000000:3.0:1788876877.855109:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1788876877.855125:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc32 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876877.855139:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc32 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876877.855152:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1788876877.855155:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06189393000 from 52 10000000:01000000:3.0:1788876877.855158:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1788876877.855213:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876877.855221:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876877.855232:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876877.855265:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511285888:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788876877.856052:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0619121e900 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.862414:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511285888:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876877.862455:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc39 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.862470:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313afc39 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876877.863450:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286016:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.863705:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.884322:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286016:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.885362:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286144:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.885749:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.896356:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286144:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:1.0:1788876877.905316:0:38759:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 51 00000020:01000000:2.0:1788876877.906336:0:15194:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 52-51 (rc=0) 10000000:01000000:2.0:1788876877.906364:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1788876877.906410:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc39 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.906428:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc39 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876877.906442:0:15194:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1788876877.906446:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06189393000 from 7 10000000:01000000:2.0:1788876877.906449:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1788876877.906524:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.906535:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.906550:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876877.906587:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286272:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.906715:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.920719:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286272:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876877.920766:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc40 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.920783:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313afc40 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876877.921011:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286400:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.921314:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.957300:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286400:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.957433:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286528:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.957669:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10e40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.963458:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286528:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1788876877.966815:0:38760:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 10000000:01000000:2.0:1788876877.976216:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:2.0:1788876877.976228:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffffa0618906b380 from 1 10000000:01000000:2.0:1788876877.976235:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1788876877.976304:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.976319:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.976331:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876877.976365:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286656:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.976989:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.990901:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286656:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876877.991149:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc47 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.991166:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313afc47 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876877.991286:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286784:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876877.991426:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876877.995817:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511286784:192.168.203.111@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1788876877.996037:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1788876877.996106:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc47 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876877.996120:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313afc47 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876878.589963:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876879.615416:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788876880.862609:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888aca80 x1875771511278464/t81604378631(81604378631) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876891 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.862623:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876880.862627:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 81604378630 gen 1 00000100:00100000:2.0:1788876880.862632:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888aca80 x1875771511278464/t81604378631(81604378631) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876891 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.862639:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 81604378630 last 81604378631 00000100:00080000:2.0:1788876880.862648:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa06188870a80 x1875771511278976/t81604378632(81604378632) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876854 ref 1 fl New:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876880.862659:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511278464:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876880.862680:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511278976:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876880.865799:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 81604378631 gen 1 00000100:00100000:2.0:1788876880.865828:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188870a80 x1875771511278976/t81604378632(81604378632) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876896 ref 2 fl Rpc:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.865841:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa06188870a80 x1875771511278976/t81604378632(81604378632) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876896 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.865850:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876880.865853:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 81604378631 gen 1 00000100:00100000:2.0:1788876880.865857:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188870a80 x1875771511278976/t81604378632(81604378632) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876896 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.865864:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 81604378631 last 81604378632 00000100:00080000:2.0:1788876880.865872:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888ac380 x1875771511280640/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876861 ref 1 fl New:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788876880.865884:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511278976:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876880.865901:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511280640:192.168.203.111@tcp:36:multiop.0 00000100:00080000:2.0:1788876880.870172:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888ac380 x1875771511280640/t81604378633(81604378633) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876896 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'multiop.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.870187:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876880.870190:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 81604378631 gen 1 00000100:00100000:2.0:1788876880.870207:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188870a80 x1875771511278976/t81604378632(81604378632) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876896 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.870216:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 81604378631 last 81604378633 00000100:00080000:2.0:1788876880.870223:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876861 ref 1 fl New:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1788876880.870234:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511280640:192.168.203.111@tcp:36:multiop.0 00000100:00100000:2.0:1788876880.870252:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511281024:192.168.203.111@tcp:101:multiop.0 00000100:00080000:2.0:1788876880.880284:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876896 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'multiop.0' uid:0 gid:0 projid:0 00000002:00080000:2.0:1788876880.880303:0:2394:0:(mdc_request.c:696:mdc_replay_open()) updating handle from 0xf5bf0f7e313afa2c to 0xf5bf0f7e313afca2 00000100:00020000:2.0:1788876880.880307:0:2394:0:(client.c:3449:ptlrpc_replay_interpret()) @@@ status 301, old was 0 req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876896 ref 2 fl Interpret:RPQU/604/0 rc 301/301 job:'multiop.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1788876880.936437:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788876880.936445:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 81604378631 gen 1 00000100:00100000:2.0:1788876880.936461:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188870a80 x1875771511278976/t81604378632(81604378632) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876896 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788876880.936471:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 81604378631 last 81604378634 00000100:00080000:2.0:1788876880.936475:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:1.0:1788876880.939824:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511281024:192.168.203.111@tcp:101:multiop.0 00000100:00100000:1.0:1788876880.940786:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511286912:192.168.203.111@tcp:400:ldlm_lock_repla.0 00000100:02000000:1.0:1788876880.953023:0:2394:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffffa06189393000: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00000100:00100000:1.0:1788876880.953034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511286912:192.168.203.111@tcp:400:ldlm_lock_repla.0 00010000:00010000:2.0:1788876881.031855:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876881.032023:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876882.685617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876883.709648:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1788876885.877288:0:38815:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876885.877301:0:38815:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876885.877326:0:38815:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876885.878240:0:38815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 38815 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876885.878255:0:38815:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38815 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876885.878269:0:38815:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 38815 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876885.878298:0:38815:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287552:192.168.203.111@tcp:101:lfs.0 00000100:00100000:1.0:1788876885.878428:0:38815:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187aae600 going to sleep for 11 seconds 00000100:00100000:3.0:1788876885.891680:0:38815:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287552:192.168.203.111@tcp:101:lfs.0 00010000:00010000:3.0:1788876885.891717:0:38815:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891732:0:38815:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891748:0:38815:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891761:0:38815:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891778:0:38815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891789:0:38815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891799:0:38815:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876885.891821:0:38815:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869b100 x1875771511287552/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876901 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876885.891838:0:38815:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891854:0:38815:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876885.891862:0:38815:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876885.891946:0:38815:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876885.891956:0:38815:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:3.0:1788876885.891964:0:38815:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876885.891967:0:38815:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac0dd3cb8 00010000:00010000:3.0:1788876885.891973:0:38815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876885.891984:0:38815:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876885.892997:0:38815:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1788876885.893009:0:38815:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876885.893036:0:38815:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876885.893058:0:38815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876885.893076:0:38815:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876885.893093:0:38815:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876885.893102:0:38815:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876885.893106:0:38815:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:3.0:1788876885.893114:0:38815:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac0dd3cb8 00010000:00010000:3.0:1788876885.893119:0:38815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876885.893131:0:38815:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876885.893333:0:38815:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876885.893347:0:38815:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876885.893359:0:38815:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876885.893370:0:38815:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876885.893377:0:38815:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876885.893425:0:38815:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876885.893431:0:38815:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:3.0:1788876885.893437:0:38815:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac0dd3bb0 00010000:00010000:3.0:1788876885.893441:0:38815:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876885.893452:0:38815:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876885.893466:0:38815:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:3.0:1788876885.893478:0:38815:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:3.0:1788876885.893613:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.893635:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00800000:00000004:3.0:1788876885.893646:0:38815:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1829, max_age 1876 00000100:00100000:3.0:1788876885.893702:0:38815:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287680:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788876885.893822:0:38815:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876885.899524:0:38815:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287680:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876885.899553:0:38815:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023705/1024000 00000080:00200080:3.0:1788876885.899628:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.899645:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00800000:00000004:3.0:1788876885.899652:0:38815:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1829, max_age 1876 00000100:00100000:3.0:1788876885.899685:0:38815:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287808:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788876885.899788:0:38815:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876885.907148:0:38815:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287808:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876885.907188:0:38815:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:3.0:1788876885.907257:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907272:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907279:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907283:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907288:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907291:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907297:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907301:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907305:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907309:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907313:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907316:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907320:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907323:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907327:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907331:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907335:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907339:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907343:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907346:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907351:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907354:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907358:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907362:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907366:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907369:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876885.907373:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:3.0:1788876885.907377:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907580:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907590:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907596:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907600:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907604:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907608:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907612:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907616:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907621:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907624:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907629:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907632:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907636:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907640:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907643:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907647:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907651:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907654:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907657:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907660:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907663:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907666:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907669:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907673:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907676:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907679:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907683:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907686:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907690:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907694:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907698:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907702:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907706:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907710:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907714:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907717:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907721:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907725:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907729:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907732:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907736:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907740:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907744:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907747:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907751:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907755:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907759:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907762:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907766:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907770:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907774:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907777:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907781:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907784:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907788:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907792:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907796:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907800:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907804:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907808:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907812:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907816:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907820:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907824:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907828:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907831:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907835:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907839:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907843:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907846:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907851:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907854:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907858:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907862:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907866:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907869:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907873:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907877:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907881:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907885:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907889:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907893:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907897:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907900:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907904:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907908:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907912:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907915:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907919:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907922:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907926:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907929:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907933:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907936:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907940:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907943:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907947:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00800000:00000080:2.0:1788876885.907950:0:38815:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.907954:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.907960:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00020000:00000004:2.0:1788876885.907968:0:38815:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1829, max_age 1876 00000100:00100000:2.0:1788876885.908004:0:38815:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287936:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876885.908075:0:38815:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e103c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876885.910306:0:38815:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511287936:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876885.910332:0:38815:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876885.910452:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.910468:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00020000:00000004:2.0:1788876885.910475:0:38815:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1829, max_age 1876 00000100:00100000:2.0:1788876885.910502:0:38815:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511288064:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876885.910550:0:38815:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e103c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876885.914819:0:38815:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:38815:1875771511288064:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876885.914849:0:38815:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788876885.914919:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914935:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914941:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914946:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914950:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914954:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914958:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914962:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914966:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914969:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914972:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914976:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914980:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914983:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914987:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914991:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.914995:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.914998:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915002:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915006:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915010:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915014:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915018:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915021:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915025:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915029:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915032:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915036:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915039:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915042:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915046:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915049:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915053:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915056:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915059:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915063:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915067:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915070:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915074:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915077:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915081:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915085:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915089:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915093:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915097:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915100:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915104:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915107:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915111:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915115:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915119:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915123:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915127:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915131:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915135:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915138:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915142:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915146:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915150:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915154:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915158:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915161:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915165:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915180:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915185:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915189:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915193:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915197:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915201:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915205:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915209:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915213:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915217:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915220:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915224:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915227:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915230:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915233:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915237:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915240:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915244:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915248:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915252:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915256:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915259:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915263:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915266:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915269:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915273:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915275:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915279:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915281:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915285:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915288:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915291:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915294:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915298:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915301:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915305:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915308:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915312:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915315:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915318:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915321:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915325:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915329:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915333:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915337:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915341:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915344:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915347:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915351:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915355:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915358:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915362:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915366:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915370:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915374:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915378:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915423:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915429:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915433:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915436:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915440:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200080:2.0:1788876885.915444:0:38815:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd158e030 00020000:00000080:2.0:1788876885.915447:0:38815:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e07400 uarg=0000000000000000 00000080:00200008:2.0:1788876885.915460:0:38815:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000100:00100000:3.0:1788876886.150633:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1788876886.150657:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511288192:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876886.168228:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511288192:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876886.784598:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876889.853593:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876890.882216:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876891.262812:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511288320:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876891.265313:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1788876891.265340:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511288448:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876891.265586:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511288704:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876891.265621:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788876891.265627:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511288576:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876891.265653:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511288832:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876891.266880:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511288320:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876891.269578:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511288704:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876891.273518:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511288832:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876891.279200:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511288576:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876891.279318:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511288448:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876892.925810:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1788876893.554027:0:39097:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg311-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 00010000:00010000:2.0:1788876893.949813:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1788876895.407619:0:39246:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:3.0:1788876895.997622:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876897.025323:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1788876897.554710:0:39284:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876897.554723:0:39284:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876897.554746:0:39284:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.554765:0:39284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876897.554781:0:39284:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876897.554790:0:39284:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.554798:0:39284:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876897.554802:0:39284:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:2.0:1788876897.554810:0:39284:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac0d87cb8 00010000:00010000:2.0:1788876897.554813:0:39284:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876897.554823:0:39284:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:2.0:1788876897.556753:0:39284:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788876897.556763:0:39284:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876897.556784:0:39284:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.556798:0:39284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876897.556811:0:39284:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876897.556818:0:39284:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.556826:0:39284:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876897.556829:0:39284:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:2.0:1788876897.556836:0:39284:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac0d87cb8 00010000:00010000:2.0:1788876897.556840:0:39284:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876897.556848:0:39284:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876897.557000:0:39284:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876897.557009:0:39284:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.557017:0:39284:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876897.557026:0:39284:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876897.557033:0:39284:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.557037:0:39284:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876897.557040:0:39284:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:2.0:1788876897.557045:0:39284:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac0d87bb0 00010000:00010000:2.0:1788876897.557048:0:39284:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876897.557056:0:39284:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1788876897.557064:0:39284:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788876897.557073:0:39284:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788876897.558645:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:2.0:1788876897.558668:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00800000:00000004:2.0:1788876897.558676:0:39284:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1877, max_age 1888 00000100:00100000:2.0:1788876897.558719:0:39284:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289600:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876897.558846:0:39284:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10780 going to sleep for 11 seconds 00000100:00100000:3.0:1788876897.566329:0:39284:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289600:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876897.566357:0:39284:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023705/1024000 00000080:00200080:3.0:1788876897.566458:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.566474:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00800000:00000004:3.0:1788876897.566481:0:39284:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1877, max_age 1888 00000100:00100000:3.0:1788876897.566515:0:39284:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289728:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788876897.566579:0:39284:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618600c9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876897.570742:0:39284:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289728:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876897.570769:0:39284:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:3.0:1788876897.570823:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570850:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570858:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570863:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570868:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570872:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570876:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570880:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570884:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570888:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570892:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570895:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570899:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570903:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570907:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570910:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570914:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570918:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570921:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570925:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570929:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570933:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570936:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570940:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570944:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570947:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570961:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570965:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570970:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570974:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570978:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570982:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570986:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570990:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.570994:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.570998:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571002:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571005:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571010:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571013:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571017:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571021:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571025:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571028:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571033:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571036:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571041:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571045:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571049:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571053:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571057:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571060:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571064:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571068:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571072:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571076:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571080:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571084:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571088:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571092:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571096:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571100:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571104:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571107:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571121:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571126:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571130:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571133:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571137:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571141:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571144:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571146:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571150:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571154:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571157:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571160:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571164:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571167:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571171:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571175:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571178:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571182:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571185:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571188:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571192:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571195:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571199:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571202:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571206:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571210:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571214:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571218:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.571221:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:3.0:1788876897.571225:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574482:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574504:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574511:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574516:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574520:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574523:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574527:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574531:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574535:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574538:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574542:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574546:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574550:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574553:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574557:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574560:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574564:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574568:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574572:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574575:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574578:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574582:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574585:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574589:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574592:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574596:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574599:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574603:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574607:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574610:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574614:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00800000:00000080:1.0:1788876897.574617:0:39284:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876897.574621:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:1.0:1788876897.574626:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00020000:00000004:1.0:1788876897.574633:0:39284:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1877, max_age 1888 00000100:00100000:1.0:1788876897.574673:0:39284:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289856:192.168.203.111@tcp:13:lfs.0 00000100:00100000:1.0:1788876897.575465:0:39284:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187ae1c00 going to sleep for 11 seconds 00000100:00100000:3.0:1788876897.581753:0:39284:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289856:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876897.581782:0:39284:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788876897.581859:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.581876:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00020000:00000004:3.0:1788876897.581884:0:39284:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1877, max_age 1888 00000100:00100000:3.0:1788876897.581919:0:39284:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289984:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876897.581995:0:39284:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618600c9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876897.595855:0:39284:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39284:1875771511289984:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876897.595885:0:39284:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788876897.595956:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.595973:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.595980:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.595985:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.595989:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.595993:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.595997:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596001:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596005:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596009:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596013:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596017:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596021:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596025:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596029:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596033:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596037:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596040:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596044:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596048:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596051:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596055:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596059:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596062:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596067:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596071:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596076:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596079:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596083:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596086:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596091:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596094:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596098:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596101:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596105:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596109:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596123:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596128:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596132:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596136:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596140:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596144:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596148:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596151:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596156:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596159:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596163:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596166:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596170:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596175:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596179:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596182:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596187:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596190:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596194:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596198:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596202:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596205:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596209:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596213:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596217:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596220:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596224:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596346:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596354:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596357:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596361:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596365:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596369:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596373:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596377:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596403:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596410:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596416:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596420:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596424:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596428:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596432:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596436:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596439:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596443:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596447:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596451:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596455:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596460:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596463:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596467:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596470:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596474:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596477:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596482:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596485:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596489:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596493:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596497:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596500:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596504:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596508:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596512:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596517:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596520:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596591:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596598:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596602:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596606:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596610:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596614:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596618:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596621:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596624:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596628:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596631:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596635:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596638:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596642:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596645:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596649:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596653:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596657:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596661:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596665:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596669:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596673:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596677:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200080:3.0:1788876897.596681:0:39284:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffde371ee90 00020000:00000080:3.0:1788876897.596685:0:39284:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389c00 uarg=0000000000000000 00000080:00200008:3.0:1788876897.596699:0:39284:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:3.0:1788876897.681748:0:39300:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876897.681775:0:39300:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876897.681795:0:39300:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876897.681811:0:39300:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788876897.681820:0:39300:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876897.681827:0:39300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876897.681830:0:39300:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:3.0:1788876897.681838:0:39300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac156fb58 00010000:00010000:3.0:1788876897.681842:0:39300:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876897.681850:0:39300:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1788876897.681865:0:39300:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876897.681880:0:39300:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=9 00000080:00200000:3.0:1788876897.681885:0:39300:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876897.681900:0:39300:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f8.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876897.681943:0:39300:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39300 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.681955:0:39300:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39300 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.681966:0:39300:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39300 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876897.681991:0:39300:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:39300:1875771511290112:192.168.203.111@tcp:101:ls.0 00000100:00100000:3.0:1788876897.682073:0:39300:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618600c9c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876897.696195:0:39300:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:39300:1875771511290112:192.168.203.111@tcp:101:ls.0 00010000:00010000:3.0:1788876897.696237:0:39300:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696260:0:39300:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696269:0:39300:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696281:0:39300:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696293:0:39300:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696306:0:39300:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696318:0:39300:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696326:0:39300:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876897.696337:0:39300:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188873100 x1875771511290112/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788876913 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876897.696351:0:39300:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696362:0:39300:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/272 (0 0) 00000080:00200000:3.0:1788876897.696369:0:39300:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788876897.696422:0:39300:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x110:0x0], updating i_size 0 00000080:00200000:3.0:1788876897.696426:0:39300:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x110:0x0](ffffa06198289148): rc = 0 00000080:00200000:3.0:1788876897.696446:0:39300:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x110:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1788876897.696452:0:39300:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) for lock 0x7d14dcdc370489ac 00000080:00010000:3.0:1788876897.696470:0:39300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876897.696474:0:39300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ac from it ffffb95ac156fb20 00010000:00010000:3.0:1788876897.696478:0:39300:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876897.696487:0:39300:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876897.696533:0:39300:0:(file.c:6296:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffffa06198289148), request_mask 2, flags 0, foreign 0 00000080:00200000:3.0:1788876897.696541:0:39300:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffffa06198289148),name=f8.replay-single 00000002:00010000:3.0:1788876897.696562:0:39300:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876897.696571:0:39300:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876897.696581:0:39300:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876897.696588:0:39300:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876897.696593:0:39300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876897.696597:0:39300:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) for lock 0x7d14dcdc370489ac 00000080:00010000:3.0:1788876897.696600:0:39300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ac from it ffffb95ac156fcb8 00010000:00010000:3.0:1788876897.696603:0:39300:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876897.696611:0:39300:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876897.696641:0:39300:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876897.696649:0:39300:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876897.696656:0:39300:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1788876897.696664:0:39300:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1788876897.696670:0:39300:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876897.696676:0:39300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876897.696679:0:39300:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:3.0:1788876897.696682:0:39300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac156fb58 00010000:00010000:3.0:1788876897.696685:0:39300:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1788876897.696692:0:39300:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1788876897.696701:0:39300:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876897.696707:0:39300:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=8 00000080:00200008:3.0:1788876897.696712:0:39300:0:(file.c:6296:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffffa06198289148), request_mask 2, flags 0, foreign 0 00000080:00200000:3.0:1788876897.696716:0:39300:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffffa06198289148),name=f8.replay-single 00000002:00010000:3.0:1788876897.696723:0:39300:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876897.696730:0:39300:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876897.696739:0:39300:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876897.696746:0:39300:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876897.696749:0:39300:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876897.696752:0:39300:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) for lock 0x7d14dcdc370489ac 00000080:00010000:3.0:1788876897.696755:0:39300:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ac from it ffffb95ac156fcb8 00010000:00010000:3.0:1788876897.696758:0:39300:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876897.696766:0:39300:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876897.715120:0:39302:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876897.715151:0:39302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.715170:0:39302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1788876897.715185:0:39302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1788876897.715194:0:39302:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.715204:0:39302:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876897.715207:0:39302:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:2.0:1788876897.715214:0:39302:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac152fb58 00010000:00010000:2.0:1788876897.715218:0:39302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1788876897.715448:0:39302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1788876897.715465:0:39302:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876897.715473:0:39302:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=8 00000080:00200008:2.0:1788876897.715483:0:39302:0:(file.c:6296:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffffa06198289148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876897.715488:0:39302:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffffa06198289148),name=f8.replay-single 00000002:00010000:2.0:1788876897.715501:0:39302:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.715509:0:39302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876897.715519:0:39302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876897.715526:0:39302:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.716013:0:39302:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) 00000080:00010000:2.0:1788876897.716018:0:39302:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) for lock 0x7d14dcdc370489ac 00000080:00010000:2.0:1788876897.716023:0:39302:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ac from it ffffb95ac152fcb8 00010000:00010000:2.0:1788876897.716027:0:39302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876897.716036:0:39302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:2.0:1788876897.719205:0:38489:0:(file.c:479:ll_file_release()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffffa06198289148), flags 100000 00000002:00100000:2.0:1788876897.719298:0:38489:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876896 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788876897.719313:0:38489:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788876897.719318:0:38489:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888aed80 x1875771511290368/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788876897.719342:0:38489:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511290368:192.168.203.111@tcp:35:multiop.0 00000100:00100000:2.0:1788876897.719450:0:38489:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788876897.723934:0:38489:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job multiop:f0204769-56bd-4f5c-ab97-74c450c9e072:38489:1875771511290368:192.168.203.111@tcp:35:multiop.0 00000002:00100000:2.0:1788876897.723980:0:38489:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa061888ad880 x1875771511281024/t81604378634(81604378634) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/608 e 0 to 0 dl 1788876896 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'multiop.0' uid:0 gid:0 projid:0 00000080:00200000:2.0:1788876897.732420:0:39304:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876897.732453:0:39304:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.732474:0:39304:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1788876897.732490:0:39304:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:2.0:1788876897.732499:0:39304:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.732507:0:39304:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876897.732510:0:39304:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:2.0:1788876897.732518:0:39304:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac153fb58 00010000:00010000:2.0:1788876897.732522:0:39304:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:2.0:1788876897.732531:0:39304:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1788876897.732548:0:39304:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876897.732555:0:39304:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=8 00000080:00200008:2.0:1788876897.732562:0:39304:0:(file.c:6296:ll_getattr_dentry()) START file f8.replay-single:[0x200001b71:0x110:0x0](ffffa06198289148), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876897.732567:0:39304:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x110:0x0](ffffa06198289148),name=f8.replay-single 00000002:00010000:2.0:1788876897.732578:0:39304:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x110:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.732585:0:39304:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876897.732594:0:39304:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876897.732602:0:39304:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.732606:0:39304:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) 00000080:00010000:2.0:1788876897.732609:0:39304:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x110:0x0](ffffa06198289148) for lock 0x7d14dcdc370489ac 00000080:00010000:2.0:1788876897.732612:0:39304:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ac from it ffffb95ac153fcb8 00010000:00010000:2.0:1788876897.732615:0:39304:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876897.732624:0:39304:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876897.732650:0:39304:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876897.732657:0:39304:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.732663:0:39304:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1788876897.732672:0:39304:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:2.0:1788876897.732678:0:39304:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.732682:0:39304:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876897.732685:0:39304:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:2.0:1788876897.732688:0:39304:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac153fc10 00010000:00010000:2.0:1788876897.732691:0:39304:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:2.0:1788876897.732699:0:39304:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1788876897.732708:0:39304:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876897.732714:0:39304:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f8.replay-single, flags=0 00000080:00200000:2.0:1788876897.732718:0:39304:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876897.732724:0:39304:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876897.732729:0:39304:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1788876897.732737:0:39304:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:2.0:1788876897.732744:0:39304:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876897.732747:0:39304:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876897.732750:0:39304:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489a5 00000080:00010000:2.0:1788876897.732753:0:39304:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489a5 from it ffffb95ac153fd68 00010000:00010000:2.0:1788876897.732756:0:39304:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:2.0:1788876897.732763:0:39304:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000080:00200000:2.0:1788876897.732772:0:39304:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876897.732778:0:39304:0:(namei.c:2351:ll_unlink()) VFS Op:name=f8.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:2.0:1788876897.732801:0:39304:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1788876897.732825:0:39304:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876897.732841:0:39304:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x110:0x0]: losing layout lock 00010000:00010000:2.0:1788876897.732874:0:39304:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876897.732882:0:39304:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00010000:00010000:2.0:1788876897.732888:0:39304:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1788876897.732891:0:39304:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a200/0x7d14dcdc370489ac lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x110:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313afe8c expref: -99 pid: 39300 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876897.732907:0:39304:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902fc00/0x7d14dcdc370489a5 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313afe70 expref: -99 pid: 38815 timeout: 0 lvb_type: 0 lru_score: 12 lru_type: 1 00000020:00100000:2.0:1788876897.732926:0:39304:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876897.732938:0:39304:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ad880 x1875771511290624/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876897.732962:0:39304:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39304:1875771511290624:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876897.733042:0:39304:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10c00 going to sleep for 11 seconds 00000100:00100000:2.0:1788876897.753768:0:39304:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39304:1875771511290624:192.168.203.111@tcp:36:rm.0 00000080:00200000:2.0:1788876897.753830:0:39304:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x110:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788876897.753839:0:39304:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x110:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788876897.753847:0:39304:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x110:0x0](ffffa06198289148) 00010000:00010000:2.0:1788876898.046067:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876899.069639:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876900.095628:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876901.117716:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876901.502676:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511290880:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876901.502762:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511291136:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876901.502794:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1788876901.502800:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511290752:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876901.502829:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511291008:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876901.502851:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511291264:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876901.517230:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511290752:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876901.517253:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511291008:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876901.517264:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511291264:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876901.517276:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511290880:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876901.517286:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511291136:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876903.165547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876905.216304:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1788876905.784248:0:39877:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 9: |X| create (same inum/gen) ====== 10:15:04 (1788876904) 00010000:00010000:3.0:1788876906.240557:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876906.627601:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876906.627625:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511291392:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.627758:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291520:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.627815:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876906.627821:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291648:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.627865:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291776:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.627920:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788876906.627926:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291904:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.633154:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291904:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.633172:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291776:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.633184:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291648:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.633193:0:2397:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 85899345922 gen 1 00000100:00100000:3.0:1788876906.633211:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511291520:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876906.645094:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511291392:192.168.203.111@tcp:400:kworker.0 00000080:00200008:1.0:1788876906.797848:0:39898:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876906.797863:0:39898:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876906.797886:0:39898:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876906.797948:0:39898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.797959:0:39898:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39898 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.797969:0:39898:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39898 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876906.797992:0:39898:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292032:192.168.203.111@tcp:101:lfs.0 00000100:00100000:1.0:1788876906.798188:0:39898:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187ae10c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876906.811519:0:39898:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292032:192.168.203.111@tcp:101:lfs.0 00010000:00010000:1.0:1788876906.811546:0:39898:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811558:0:39898:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811571:0:39898:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811582:0:39898:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811600:0:39898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811608:0:39898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811617:0:39898:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788876906.811629:0:39898:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188699500 x1875771511292032/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876922 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788876906.811643:0:39898:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811656:0:39898:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788876906.811662:0:39898:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788876906.811688:0:39898:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788876906.811694:0:39898:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489b3 00000080:00010000:1.0:1788876906.811699:0:39898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876906.811702:0:39898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489b3 from it ffffb95ac1e0fcb8 00010000:00010000:1.0:1788876906.811706:0:39898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876906.811715:0:39898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1788876906.812378:0:39898:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1788876906.812425:0:39898:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876906.812442:0:39898:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876906.812452:0:39898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876906.812463:0:39898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788876906.812470:0:39898:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876906.812474:0:39898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876906.812477:0:39898:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489b3 00000080:00010000:1.0:1788876906.812481:0:39898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489b3 from it ffffb95ac1e0fcb8 00010000:00010000:1.0:1788876906.812484:0:39898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788876906.812491:0:39898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876906.812593:0:39898:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876906.812603:0:39898:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876906.812611:0:39898:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876906.812619:0:39898:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788876906.812630:0:39898:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876906.812634:0:39898:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876906.812637:0:39898:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489b3 00000080:00010000:1.0:1788876906.812641:0:39898:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489b3 from it ffffb95ac1e0fbb0 00010000:00010000:1.0:1788876906.812644:0:39898:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876906.812651:0:39898:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876906.812661:0:39898:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:1.0:1788876906.812668:0:39898:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:1.0:1788876906.812768:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.812783:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00800000:00000004:1.0:1788876906.812789:0:39898:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1889, max_age 1897 00000100:00100000:1.0:1788876906.812824:0:39898:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292160:192.168.203.111@tcp:41:lfs.0 00000100:00100000:1.0:1788876906.813001:0:39898:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187ae10c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876906.823927:0:39898:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292160:192.168.203.111@tcp:41:lfs.0 00800000:00000004:1.0:1788876906.823956:0:39898:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023706/1024000 00000080:00200080:1.0:1788876906.824041:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.824058:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00800000:00000004:1.0:1788876906.824065:0:39898:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1889, max_age 1897 00000100:00100000:1.0:1788876906.824108:0:39898:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292288:192.168.203.111@tcp:41:lfs.0 00000100:00100000:1.0:1788876906.824180:0:39898:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187ae10c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876906.826083:0:39898:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292288:192.168.203.111@tcp:41:lfs.0 00800000:00000004:1.0:1788876906.826109:0:39898:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:1.0:1788876906.826164:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826179:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826187:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826191:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826195:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826199:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826203:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826207:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826211:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826215:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826219:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826223:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826227:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826230:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826234:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826238:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826242:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826245:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826249:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826253:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826257:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826260:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826264:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826268:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826272:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826276:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826280:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826284:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826289:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826293:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826297:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826300:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826304:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826307:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826311:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826314:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826317:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826320:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826324:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826327:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826330:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826333:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826336:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826339:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826342:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826345:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826348:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826352:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826356:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826360:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826364:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00800000:00000080:1.0:1788876906.826367:0:39898:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1fa8000 uarg=0000000000000000 00000080:00200080:1.0:1788876906.826371:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00000100:00100000:3.0:1788876906.828790:0:39898:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292416:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876906.828814:0:39898:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788876906.828872:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.828887:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00020000:00000004:3.0:1788876906.828894:0:39898:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1889, max_age 1897 00000100:00100000:3.0:1788876906.828930:0:39898:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292544:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876906.828985:0:39898:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a0c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876906.838767:0:39898:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:39898:1875771511292544:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876906.838808:0:39898:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788876906.838884:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838902:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838918:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838924:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838928:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838931:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838936:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838940:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838943:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838948:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838952:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838956:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838959:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838963:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838966:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838970:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838974:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838977:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838981:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838985:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838988:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.838992:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.838996:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839000:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839004:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839008:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839013:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839016:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839020:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839024:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839028:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839032:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839035:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839039:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839044:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839048:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839052:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839056:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839060:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839063:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839078:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839082:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839087:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839090:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839094:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839098:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839102:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839106:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839109:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839113:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839117:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839120:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839124:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839128:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839132:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839135:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839139:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839142:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839146:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839151:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839155:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839158:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839162:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839166:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839170:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839173:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839178:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839181:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839185:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839188:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839192:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839196:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839200:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839203:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839207:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839210:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839214:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839217:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839221:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839224:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839228:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839231:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839235:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839238:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839243:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839247:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839251:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839254:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839258:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839261:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839265:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839268:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839272:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839275:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839279:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839283:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839287:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839290:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839294:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839297:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839301:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839305:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839309:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839313:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839317:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839320:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839324:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839328:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839331:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839335:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839339:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839343:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839346:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839350:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839354:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839357:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839360:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839364:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839368:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839371:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839427:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839433:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839438:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839442:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200080:3.0:1788876906.839446:0:39898:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fffd97f82f0 00020000:00000080:3.0:1788876906.839449:0:39898:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838b400 uarg=0000000000000000 00000080:00200008:3.0:1788876906.839463:0:39898:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:3.0:1788876906.890758:0:39905:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876906.890792:0:39905:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876906.890813:0:39905:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876906.890829:0:39905:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876906.890842:0:39905:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876906.890850:0:39905:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876906.890854:0:39905:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489b3 00000080:00010000:3.0:1788876906.890862:0:39905:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489b3 from it ffffb95ac1b5fbd0 00010000:00010000:3.0:1788876906.890866:0:39905:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876906.890874:0:39905:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788876906.890891:0:39905:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876906.890905:0:39905:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:3.0:1788876906.890909:0:39905:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876906.890915:0:39905:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876906.890921:0:39905:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876906.890928:0:39905:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788876906.890934:0:39905:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876906.890938:0:39905:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876906.890941:0:39905:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489b3 00000080:00010000:3.0:1788876906.890944:0:39905:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489b3 from it ffffb95ac1b5fcd0 00010000:00010000:3.0:1788876906.890947:0:39905:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788876906.890960:0:39905:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876906.890968:0:39905:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876906.890974:0:39905:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876906.890980:0:39905:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876906.890986:0:39905:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876906.890994:0:39905:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788876906.891000:0:39905:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876906.891003:0:39905:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876906.891006:0:39905:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489b3 00000080:00010000:3.0:1788876906.891009:0:39905:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489b3 from it ffffb95ac1b5fdb0 00010000:00010000:3.0:1788876906.891012:0:39905:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788876906.891020:0:39905:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788876906.891028:0:39905:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788876906.891033:0:39905:0:(namei.c:2062:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:3.0:1788876906.891037:0:39905:0:(namei.c:2017:ll_mknod()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:3.0:1788876906.891066:0:39905:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876906.891137:0:39905:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876906.891144:0:39905:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788876906.891147:0:39905:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f600/0x7d14dcdc370489b3 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313afeb6 expref: -99 pid: 39898 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:3.0:1788876906.891171:0:39905:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788876906.891184:0:39905:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188873100 x1875771511292800/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876906.891207:0:39905:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:39905:1875771511292800:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:3.0:1788876906.891301:0:39905:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a0c0 going to sleep for 11 seconds 00000080:00200000:3.0:1788876906.913974:0:39908:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876906.914003:0:39908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876906.914058:0:39908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.914080:0:39908:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.914090:0:39908:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39908 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876906.914115:0:39908:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39908:1875771511292928:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876906.914199:0:39908:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876906.923455:0:39908:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39908:1875771511292928:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876906.923483:0:39908:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923494:0:39908:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923505:0:39908:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923514:0:39908:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923526:0:39908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923533:0:39908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923540:0:39908:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876906.923552:0:39908:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188871180 x1875771511292928/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876922 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876906.923565:0:39908:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923576:0:39908:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876906.923582:0:39908:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876906.923608:0:39908:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876906.923614:0:39908:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489ba 00000080:00010000:3.0:1788876906.923619:0:39908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876906.923621:0:39908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ba from it ffffb95ac0d17b58 00010000:00010000:3.0:1788876906.923624:0:39908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923632:0:39908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876906.923655:0:39908:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876906.923674:0:39908:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:3.0:1788876906.923679:0:39908:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876906.923700:0:39908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: fsa-oleg311-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876906.923737:0:39908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923744:0:39908:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39908 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.923753:0:39908:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39908 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876906.923773:0:39908:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39908:1875771511293056:192.168.203.111@tcp:101:rm.0 00000100:00100000:3.0:1788876906.923838:0:39908:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876906.939307:0:39908:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39908:1875771511293056:192.168.203.111@tcp:101:rm.0 00010000:00010000:3.0:1788876906.939341:0:39908:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939400:0:39908:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939410:0:39908:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939421:0:39908:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939435:0:39908:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939448:0:39908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939456:0:39908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939464:0:39908:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876906.939476:0:39908:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188871180 x1875771511293056/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 600/584 e 0 to 0 dl 1788876922 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876906.939488:0:39908:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876906.939503:0:39908:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/273 (0 0) 00000080:00200000:3.0:1788876906.939510:0:39908:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788876906.939612:0:39908:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x111:0x0], updating i_size 0 00000080:00200000:2.0:1788876906.939617:0:39908:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x111:0x0](ffffa061981be3c8): rc = 0 00000080:00200000:2.0:1788876906.939633:0:39908:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x111:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1788876906.939639:0:39908:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x111:0x0](ffffa061981be3c8) for lock 0x7d14dcdc370489c1 00000080:00010000:2.0:1788876906.939651:0:39908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x111:0x0](ffffa061981be3c8) 00000080:00010000:2.0:1788876906.939654:0:39908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489c1 from it ffffb95ac0d17b20 00010000:00010000:2.0:1788876906.939658:0:39908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876906.939668:0:39908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788876906.939708:0:39908:0:(file.c:6296:ll_getattr_dentry()) START file fsa-oleg311-client.virtnet:[0x200001b71:0x111:0x0](ffffa061981be3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788876906.939716:0:39908:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x111:0x0](ffffa061981be3c8),name=fsa-oleg311-client.virtnet 00000002:00010000:2.0:1788876906.939739:0:39908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x111:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876906.939749:0:39908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876906.939763:0:39908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876906.939770:0:39908:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876906.939787:0:39908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x111:0x0](ffffa061981be3c8) 00000080:00010000:2.0:1788876906.939790:0:39908:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x111:0x0](ffffa061981be3c8) for lock 0x7d14dcdc370489c1 00000080:00010000:2.0:1788876906.939794:0:39908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489c1 from it ffffb95ac0d17cb8 00010000:00010000:2.0:1788876906.939798:0:39908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876906.939804:0:39908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876906.939861:0:39908:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876906.939869:0:39908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876906.939876:0:39908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876906.939884:0:39908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876906.939890:0:39908:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876906.939893:0:39908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876906.939895:0:39908:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489ba 00000080:00010000:2.0:1788876906.939898:0:39908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ba from it ffffb95ac0d17c10 00010000:00010000:2.0:1788876906.939901:0:39908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876906.939919:0:39908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876906.939927:0:39908:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876906.939936:0:39908:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg311-client.virtnet, flags=0 00000080:00200000:2.0:1788876906.939940:0:39908:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876906.939947:0:39908:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876906.939952:0:39908:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876906.939960:0:39908:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876906.939965:0:39908:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876906.939969:0:39908:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876906.939971:0:39908:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489ba 00000080:00010000:2.0:1788876906.939974:0:39908:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489ba from it ffffb95ac0d17d68 00010000:00010000:2.0:1788876906.939976:0:39908:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876906.939984:0:39908:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876906.939991:0:39908:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876906.939996:0:39908:0:(namei.c:2351:ll_unlink()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:2.0:1788876906.940015:0:39908:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876906.940036:0:39908:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876906.940045:0:39908:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x111:0x0]: losing layout lock 00010000:00010000:2.0:1788876906.940084:0:39908:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876906.940092:0:39908:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876906.940097:0:39908:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1788876906.940100:0:39908:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b800/0x7d14dcdc370489c1 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x111:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313afeee expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876906.940117:0:39908:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc370489ba lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313afed2 expref: -99 pid: 39908 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1788876906.940135:0:39908:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876906.940141:0:39908:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ad500 x1875771511293312/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876906.940162:0:39908:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39908:1875771511293312:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788876906.940239:0:39908:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a006c0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876906.953203:0:39908:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:39908:1875771511293312:192.168.203.111@tcp:36:rm.0 00000080:00200000:2.0:1788876906.953270:0:39908:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x111:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788876906.953281:0:39908:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x111:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788876906.953290:0:39908:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x111:0x0](ffffa061981be3c8) 00010000:00010000:1.0:1788876908.285616:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876909.310100:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876911.360488:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876911.746538:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511293568:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876911.746625:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511293824:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876911.746659:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1788876911.746665:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511293440:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876911.746695:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511293696:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876911.746714:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511293952:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876911.755830:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511293440:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876911.755855:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511293952:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876911.755867:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511293824:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876912.381561:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876914.432107:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876914.856011:0:39974:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:2.0:1788876914.884130:0:39975:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876914.884162:0:39975:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876914.884228:0:39975:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39975 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.884240:0:39975:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39975 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.884251:0:39975:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39975 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876914.884279:0:39975:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:39975:1875771511294080:192.168.203.111@tcp:101:mcreate.0 00000100:00100000:2.0:1788876914.884362:0:39975:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a006c9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876914.892598:0:39975:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:39975:1875771511294080:192.168.203.111@tcp:101:mcreate.0 00010000:00010000:2.0:1788876914.892637:0:39975:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892650:0:39975:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892665:0:39975:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892679:0:39975:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892697:0:39975:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892706:0:39975:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892713:0:39975:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788876914.892726:0:39975:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888aca80 x1875771511294080/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876930 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788876914.892743:0:39975:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892757:0:39975:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788876914.892765:0:39975:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788876914.892799:0:39975:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876914.892805:0:39975:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489c8 00000080:00010000:2.0:1788876914.892811:0:39975:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876914.892815:0:39975:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489c8 from it ffffb95ac1d2fbd0 00010000:00010000:2.0:1788876914.892819:0:39975:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876914.892829:0:39975:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876914.892864:0:39975:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876914.892905:0:39975:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:2.0:1788876914.892910:0:39975:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876914.892929:0:39975:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876914.892939:0:39975:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876914.892952:0:39975:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876914.892959:0:39975:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876914.892964:0:39975:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876914.892967:0:39975:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489c8 00000080:00010000:2.0:1788876914.892970:0:39975:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489c8 from it ffffb95ac1d2fcd0 00010000:00010000:2.0:1788876914.892973:0:39975:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876914.892981:0:39975:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876914.892990:0:39975:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876914.892997:0:39975:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876914.893003:0:39975:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876914.893010:0:39975:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876914.893023:0:39975:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876914.893042:0:39975:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876914.893047:0:39975:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876914.893050:0:39975:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489c8 00000080:00010000:2.0:1788876914.893053:0:39975:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489c8 from it ffffb95ac1d2fdb0 00010000:00010000:2.0:1788876914.893057:0:39975:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876914.893065:0:39975:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876914.893073:0:39975:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876914.893079:0:39975:0:(namei.c:2062:ll_create_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:2.0:1788876914.893084:0:39975:0:(namei.c:2017:ll_mknod()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:2.0:1788876914.893111:0:39975:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876914.893152:0:39975:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876914.893159:0:39975:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876914.893162:0:39975:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031c00/0x7d14dcdc370489c8 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313aff18 expref: -99 pid: 39975 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1788876914.893188:0:39975:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876914.893196:0:39975:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888aca80 x1875771511294336/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876914.893218:0:39975:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:39975:1875771511294336:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876914.893298:0:39975:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a006c9c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876914.897189:0:39975:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:39975:1875771511294336:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:2.0:1788876914.897248:0:39975:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00200000:2.0:1788876914.897291:0:39975:0:(namei.c:2070:ll_create_nd()) VFS Op:name=f9.replay-single, unhashed 1 00000080:00200000:1.0:1788876914.925312:0:39977:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876914.925356:0:39977:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876914.925439:0:39977:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 39977 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876914.925450:0:39977:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39977 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876914.925462:0:39977:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 39977 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788876914.925493:0:39977:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:39977:1875771511294464:192.168.203.111@tcp:101:ls.0 00000100:00100000:1.0:1788876914.925593:0:39977:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06192efb480 going to sleep for 11 seconds 00000100:00100000:3.0:1788876914.930859:0:39977:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:39977:1875771511294464:192.168.203.111@tcp:101:ls.0 00010000:00010000:3.0:1788876914.930895:0:39977:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.930908:0:39977:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.930922:0:39977:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.930933:0:39977:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.930948:0:39977:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.930957:0:39977:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.930966:0:39977:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876914.930984:0:39977:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869bb80 x1875771511294464/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876930 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876914.930998:0:39977:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.931011:0:39977:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876914.931018:0:39977:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876914.931268:0:39977:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876914.931276:0:39977:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489cf 00000080:00010000:3.0:1788876914.931281:0:39977:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876914.931284:0:39977:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489cf from it ffffb95ac1cefb58 00010000:00010000:3.0:1788876914.931288:0:39977:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.931298:0:39977:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876914.931333:0:39977:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876914.931352:0:39977:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=9 00000080:00200000:3.0:1788876914.931356:0:39977:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876914.931411:0:39977:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f9.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876914.931451:0:39977:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 39977 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.931459:0:39977:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39977 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.931473:0:39977:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 39977 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876914.931493:0:39977:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:39977:1875771511294592:192.168.203.111@tcp:101:ls.0 00000100:00100000:3.0:1788876914.931554:0:39977:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a540 going to sleep for 11 seconds 00000100:00100000:3.0:1788876914.935316:0:39977:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:39977:1875771511294592:192.168.203.111@tcp:101:ls.0 00010000:00010000:3.0:1788876914.935347:0:39977:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935373:0:39977:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935408:0:39977:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935421:0:39977:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935433:0:39977:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935447:0:39977:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935455:0:39977:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935469:0:39977:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876914.935481:0:39977:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188873b80 x1875771511294592/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788876930 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876914.935495:0:39977:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935505:0:39977:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/274 (0 0) 00000080:00200000:3.0:1788876914.935512:0:39977:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788876914.935533:0:39977:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x112:0x0], updating i_size 0 00000080:00200000:3.0:1788876914.935536:0:39977:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x112:0x0](ffffa06198289148): rc = 0 00000080:00200000:3.0:1788876914.935552:0:39977:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x112:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1788876914.935556:0:39977:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) for lock 0x7d14dcdc370489d6 00000080:00010000:3.0:1788876914.935566:0:39977:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876914.935570:0:39977:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489d6 from it ffffb95ac1cefb20 00010000:00010000:3.0:1788876914.935573:0:39977:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876914.935581:0:39977:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876914.935619:0:39977:0:(file.c:6296:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffffa06198289148), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1788876914.935626:0:39977:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffffa06198289148),name=f9.replay-single 00000002:00010000:3.0:1788876914.935646:0:39977:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876914.935655:0:39977:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876914.935667:0:39977:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876914.935674:0:39977:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876914.935680:0:39977:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876914.935683:0:39977:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) for lock 0x7d14dcdc370489d6 00000080:00010000:3.0:1788876914.935687:0:39977:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489d6 from it ffffb95ac1cefcb8 00010000:00010000:3.0:1788876914.935691:0:39977:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876914.935698:0:39977:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876914.935737:0:39977:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876914.935744:0:39977:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876914.935751:0:39977:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876914.935758:0:39977:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876914.935764:0:39977:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876914.935768:0:39977:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876914.935770:0:39977:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489cf 00000080:00010000:3.0:1788876914.935773:0:39977:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489cf from it ffffb95ac1cefb58 00010000:00010000:3.0:1788876914.935776:0:39977:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876914.935783:0:39977:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876914.935792:0:39977:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876914.935798:0:39977:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=8 00000080:00200008:3.0:1788876914.935804:0:39977:0:(file.c:6296:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffffa06198289148), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1788876914.935807:0:39977:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffffa06198289148),name=f9.replay-single 00000002:00010000:3.0:1788876914.935816:0:39977:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876914.935822:0:39977:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876914.935831:0:39977:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876914.935838:0:39977:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876914.935843:0:39977:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876914.935845:0:39977:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) for lock 0x7d14dcdc370489d6 00000080:00010000:3.0:1788876914.935849:0:39977:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489d6 from it ffffb95ac1cefcb8 00010000:00010000:3.0:1788876914.935852:0:39977:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876914.935860:0:39977:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788876915.454823:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876917.501643:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876918.525566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876920.575587:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876921.597902:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876921.981900:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511295488:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876921.982024:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511295744:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876921.982061:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1788876921.982067:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511295360:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876921.982108:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511295616:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876921.982134:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511295872:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876921.988465:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511296000:192.168.203.111@tcp:38:ptlrpcd_00_01.0 00000100:00080000:2.0:1788876921.996511:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876921.996530:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876921.996536:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511296000:192.168.203.111@tcp:38:ptlrpcd_00_01.0 00010000:00010000:2.0:1788876923.653707:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876924.670764:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876926.717670:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876927.101771:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511296128:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876927.102097:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511296256:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788876927.123309:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876927.123324:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876927.123329:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511296256:192.168.203.111@tcp:38:kworker.0 00010000:00010000:3.0:1788876927.741571:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876929.794518:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876930.814697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876932.222240:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511296896:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788876932.234760:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788876932.234781:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788876932.234788:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511296896:192.168.203.111@tcp:38:kworker.0 00010000:00010000:2.0:1788876932.862131:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876933.886280:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876934.909713:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876935.933984:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876936.960069:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876937.342491:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511296768:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876937.343595:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876937.343709:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788876937.343713:0:8:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876937.343723:0:8:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1867 00000100:00080000:3.0:1788876937.343737:0:8:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788876937.343889:0:8:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511297408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876937.343987:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511297408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788876937.344166:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788876937.344170:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788876937.344172:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788876937.344176:0:8:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1924 00000100:00080000:3.0:1788876937.344182:0:8:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1788876937.344188:0:8:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788876937.344198:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788876937.344213:0:8:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511297536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876937.344345:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511297536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788876937.344823:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511297664:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876937.348497:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511297792:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876937.348587:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1788876937.348605:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511297920:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876937.350940:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511297664:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876937.351028:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876937.351035:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876937.351039:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511297536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788876937.358040:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511297920:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876937.358058:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511297792:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876937.985885:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876939.006759:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876940.029656:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876941.053680:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876942.078444:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788876942.462523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788876942.462532:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788876942.462535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788876942.462539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788876942.462543:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788876942.462553:0:8:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1929 00000100:00080000:1.0:1788876942.462564:0:8:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788876942.462571:0:8:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788876942.462583:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788876942.462613:0:8:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511298048/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876942.462803:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511298304:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876942.462878:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788876942.462885:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511298176:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876942.462921:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511298432:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876942.482174:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511298432:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876943.104202:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876944.128277:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876945.149782:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876946.181816:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876946.559158:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1788876946.563519:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511298560:192.168.203.111@tcp:17:kworker.0 00000100:00100000:2.0:1788876946.574851:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511298560:192.168.203.111@tcp:17:kworker.0 00010000:00010000:1.0:1788876947.198316:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876947.583562:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876937/real 1788876937] req@ffffa06188873b80 x1875771511297408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876947 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876947.583590:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511297408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788876947 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876947.583609:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511298688:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788876947.584532:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876947.584559:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788876947.584565:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511297408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788876947.584636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788876947.584642:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788876947.584644:0:8:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788876947.584649:0:8:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1929 00000100:00080000:1.0:1788876947.584657:0:8:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1788876947.584663:0:8:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788876947.584690:0:8:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511299200/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788876947.584857:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511299200:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788876947.605731:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) MGC192.168.203.111@tcp: connect to target with instance 0 10000000:01000000:3.0:1788876947.605757:0:2394:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1788876947.605769:0:2394:0:(import.c:1328:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.203.111@tcp) after server handle changed from 0xf5bf0f7e313afb67 to 0xf5bf0f7e313aff73 00000100:00080000:3.0:1788876947.605785:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1788876947.605792:0:2394:0:(import.c:1718:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.203.111@tcp; invalidating 00000100:00100000:1.0:1788876947.606543:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511298816:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876947.606568:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511299072:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788876947.612353:0:40201:0:(import.c:1642:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.203.111@tcp to MGS@192.168.203.111@tcp 00000100:00100000:3.0:1788876947.612365:0:40201:0:(import.c:323:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1788876947.612373:0:40201:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1788876947.612696:0:40201:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313afc39 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788876947.613468:0:40201:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313afc39 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788876947.613483:0:40201:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1788876947.614242:0:40201:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876947.614262:0:40201:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887bc00/0x7d14dcdc37048990 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313afc39 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876947.614282:0:40201:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313afc47 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788876947.614289:0:40201:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313afc47 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876947.614300:0:40201:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1788876947.614341:0:40201:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876947.614346:0:40201:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06180030800/0x7d14dcdc3704899e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313afc47 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876947.614365:0:40201:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313afc32 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788876947.614372:0:40201:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313afc32 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876947.614378:0:40201:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1788876947.614595:0:40201:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876947.614601:0:40201:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887a000/0x7d14dcdc37048989 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313afc32 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876947.614615:0:40201:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313afc40 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788876947.614623:0:40201:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313afc40 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876947.614628:0:40201:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1788876947.614923:0:40201:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788876947.614929:0:40201:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048997 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313afc40 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:2.0:1788876947.614950:0:40201:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:2.0:1788876947.614956:0:40201:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from RECOVER to FULL 10000000:01000000:2.0:1788876947.614961:0:40201:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:2.0:1788876947.615026:0:40201:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:2.0:1788876947.615035:0:40201:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) MGC192.168.203.111@tcp: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00010000:00010000:2.0:1788876948.221663:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876949.246106:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788876949.445618:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299328:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876949.445658:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa061886cea00/0x7d14dcdc370489dd lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0099 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876949.445673:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa061886cea00/0x7d14dcdc370489dd lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b0099 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876949.445783:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299456:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00100000:2.0:1788876949.445892:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876949.452347:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299456:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00080000:2.0:1788876949.452474:0:15194:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa06183c27000 already not connected (conn 140, was 0): DISCONN 00010000:00080000:2.0:1788876949.452491:0:15194:0:(ldlm_lib.c:85:import_set_conn()) imp ffffa06183c27000@lustre-MDT0000-mdc-ffffa06189393000: found existing conn 192.168.203.111@tcp, moved to head 00000100:00080000:2.0:1788876949.452498:0:15194:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788876949.452504:0:15194:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 0 00000100:00080000:2.0:1788876949.452514:0:15194:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788876949.452528:0:15194:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788876949.452575:0:15194:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511299584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 10000000:01000000:2.0:1788876949.452648:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1788876949.452661:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa061886cea00/0x7d14dcdc370489dd lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0099 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876949.452673:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa061886cea00/0x7d14dcdc370489dd lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0099 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788876949.452683:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:2.0:1788876949.452686:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06189393000 from 52 10000000:01000000:2.0:1788876949.452690:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:2.0:1788876949.452786:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876949.452796:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876949.452806:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876949.452828:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299712:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1788876949.452995:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00010000:00080000:3.0:1788876949.478030:0:2394:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa061888ac000 x1875771511299584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/416 e 0 to 0 dl 1788876959 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876949.478063:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa06189393000: connect to target with instance 21 00000100:00080000:3.0:1788876949.478091:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:3.0:1788876949.478100:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876949.478103:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 85899345922 gen 1 00000100:00100000:3.0:1788876949.478111:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188873100 x1875771511292800/t85899345923(85899345923) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876922 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876949.478120:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 85899345922 last 0 00000100:00080000:3.0:1788876949.478129:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa06188873100 x1875771511292800/t85899345923(85899345923) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876922 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876949.478144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511299584:192.168.203.111@tcp:38:ll_cfg_requeue.0 00000100:00100000:3.0:1788876949.478183:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511292800:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876949.483261:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299712:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876949.483300:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b00a0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876949.483314:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b00a0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876949.483456:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299840:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876949.483553:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876949.489594:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299840:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788876949.489697:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299968:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1788876949.489832:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64b40 going to sleep for 11 seconds 00000100:00100000:2.0:1788876949.502703:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511299968:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1788876949.507963:0:40211:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 51 00000100:00100000:2.0:1788876949.514125:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300096:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788876949.514171:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa061886cf200/0x7d14dcdc370489eb lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b00a7 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876949.514187:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa061886cf200/0x7d14dcdc370489eb lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b00a7 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788876949.514298:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300224:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876949.514759:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64480 going to sleep for 11 seconds 00000100:00100000:3.0:1788876949.545849:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300224:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876949.545948:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300352:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1788876949.546093:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a0c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876949.551089:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300352:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1788876949.553300:0:40212:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000100:00100000:3.0:1788876949.567657:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300480:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788876949.567697:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa061886cf000/0x7d14dcdc370489f2 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b00ae expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876949.567720:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa061886cf000/0x7d14dcdc370489f2 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b00ae expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876949.567825:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300608:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788876949.568033:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a0c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788876949.581945:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511300608:192.168.203.111@tcp:501:ll_cfg_requeue.0 10000000:01000000:1.0:1788876949.582082:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:1.0:1788876949.582096:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa061886cf000/0x7d14dcdc370489f2 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b00ae expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788876949.582111:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa061886cf000/0x7d14dcdc370489f2 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b00ae expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876950.270962:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876951.294763:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876952.317882:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876952.707847:0:2395:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1788876952.707881:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511300992:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788876952.708076:0:2395:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1788876952.708084:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511300864:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876952.711232:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (2 blwis in pool) 00010000:00010000:2.0:1788876952.711264:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1788876952.711272:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1788876952.711278:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876952.711280:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876952.711282:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876952.713266:0:2395:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 0 blwis in pool 00000100:00100000:1.0:1788876952.713297:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511301120:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876952.713427:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876952.714270:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876953.341698:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788876954.097929:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa06188873100 x1875771511292800/t85899345923(85899345923) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876965 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.097947:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876954.097950:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 85899345922 gen 1 00000100:00100000:3.0:1788876954.097955:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188873100 x1875771511292800/t85899345923(85899345923) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876965 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.097963:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 85899345922 last 85899345923 00000100:00080000:3.0:1788876954.097973:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888ad500 x1875771511293312/t85899345924(85899345924) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876922 ref 1 fl New:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876954.097985:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511292800:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:3.0:1788876954.098006:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511293312:192.168.203.111@tcp:36:rm.0 00000100:00080000:3.0:1788876954.117885:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888ad500 x1875771511293312/t85899345924(85899345924) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876970 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.117901:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876954.117903:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 85899345922 gen 1 00000100:00100000:3.0:1788876954.117910:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188873100 x1875771511292800/t85899345923(85899345923) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876965 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.117918:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 85899345922 last 85899345924 00000100:00080000:3.0:1788876954.117924:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888aca80 x1875771511294336/t85899345925(85899345925) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876930 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788876954.117935:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511293312:192.168.203.111@tcp:36:rm.0 00000100:00100000:3.0:1788876954.117955:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511294336:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:3.0:1788876954.133015:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 85899345923 gen 1 00000100:00100000:3.0:1788876954.133045:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888ad500 x1875771511293312/t85899345924(85899345924) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876970 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.133060:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888aca80 x1875771511294336/t85899345925(85899345925) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/448 e 0 to 0 dl 1788876970 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.133068:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:3.0:1788876954.133071:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 85899345923 gen 1 00000100:00100000:3.0:1788876954.133074:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888ad500 x1875771511293312/t85899345924(85899345924) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876970 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.133081:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 85899345923 last 85899345925 00000100:00080000:3.0:1788876954.133085:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:3.0:1788876954.151096:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511294336:192.168.203.111@tcp:36:mcreate.0 00010000:00080000:1.0:1788876954.151255:0:40250:0:(ldlm_request.c:2972:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffffa06189393000 to lustre-MDT0000_UUID@192.168.203.111@tcp 00010000:00010000:1.0:1788876954.151260:0:40250:0:(ldlm_request.c:2888:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffffa06189393000 (2) 00000080:00200000:1.0:1788876954.151295:0:40250:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x112:0x0]: losing layout lock 00010000:00010000:1.0:1788876954.151335:0:40250:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc370489d6 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000000 nid: local remote: 0xf5bf0f7e313aff50 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876954.151370:0:40250:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902f000/0x7d14dcdc370489cf lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0xf5bf0f7e313aff34 expref: -99 pid: 39977 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788876954.151408:0:40250:0:(ldlm_request.c:2901:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffffa06189393000 00000100:00080000:1.0:1788876954.152219:0:40250:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:3.0:1788876954.153221:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511301248:192.168.203.111@tcp:400:ldlm_lock_repla.0 00000100:00100000:3.0:1788876954.173102:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 85899345925 gen 1 00000100:00080000:3.0:1788876954.173131:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:3.0:1788876954.173143:0:2394:0:(client.c:3248:ptlrpc_resend_req()) @@@ going to resend req@ffffa0618869aa00 x1875771511301248/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788876970 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.173153:0:2394:0:(client.c:3257:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa0618869aa00 x1875771511301248/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788876970 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788876954.173160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00010000:00010000:2.0:1788876954.365632:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876954.751179:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301504:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.751293:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.751325:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:3.0:1788876954.751331:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511301376:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.751358:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301632:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.751377:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.755931:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511301376:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.755952:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301632:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.755962:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.755972:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301504:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876954.755982:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511301760:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876955.389582:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876956.414436:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876957.437660:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876958.461547:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1788876959.169081:0:40303:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788876959.169098:0:40303:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876959.169129:0:40303:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876959.169215:0:40303:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 40303 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.169228:0:40303:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 40303 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.169251:0:40303:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 40303 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876959.169283:0:40303:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302016:192.168.203.111@tcp:101:lfs.0 00000100:00100000:3.0:1788876959.170560:0:40303:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a0c0 going to sleep for 11 seconds 00000100:00100000:3.0:1788876959.185879:0:40303:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302016:192.168.203.111@tcp:101:lfs.0 00010000:00010000:3.0:1788876959.185913:0:40303:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.185928:0:40303:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.185941:0:40303:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.185953:0:40303:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.185966:0:40303:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.185975:0:40303:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.185983:0:40303:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876959.185997:0:40303:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188873800 x1875771511302016/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788876975 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876959.186013:0:40303:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.186029:0:40303:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788876959.186037:0:40303:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788876959.186073:0:40303:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788876959.186080:0:40303:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:3.0:1788876959.186086:0:40303:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876959.186090:0:40303:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac0c87cb8 00010000:00010000:3.0:1788876959.186094:0:40303:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876959.186103:0:40303:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876959.187098:0:40303:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1788876959.187108:0:40303:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876959.187133:0:40303:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876959.187149:0:40303:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876959.187162:0:40303:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876959.187171:0:40303:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876959.187178:0:40303:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876959.187181:0:40303:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:3.0:1788876959.187187:0:40303:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac0c87cb8 00010000:00010000:3.0:1788876959.187191:0:40303:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876959.187198:0:40303:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876959.187350:0:40303:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876959.187359:0:40303:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876959.187367:0:40303:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876959.187376:0:40303:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876959.187418:0:40303:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876959.187423:0:40303:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876959.187426:0:40303:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:3.0:1788876959.187431:0:40303:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac0c87bb0 00010000:00010000:3.0:1788876959.187435:0:40303:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876959.187443:0:40303:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788876959.187454:0:40303:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:3.0:1788876959.187462:0:40303:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:3.0:1788876959.189151:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:3.0:1788876959.189193:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00800000:00000004:3.0:1788876959.189204:0:40303:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1898, max_age 1950 00000100:00100000:3.0:1788876959.189282:0:40303:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302144:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876959.191490:0:40303:0:(client.c:2716:ptlrpc_set_wait()) set ffffa0618133a0c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788876959.208916:0:40303:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302144:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876959.208958:0:40303:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023705/1024000 00000080:00200080:2.0:1788876959.209221:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.209257:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00800000:00000004:2.0:1788876959.209267:0:40303:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1898, max_age 1950 00000100:00100000:2.0:1788876959.209321:0:40303:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302272:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788876959.209455:0:40303:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64480 going to sleep for 11 seconds 00000100:00100000:2.0:1788876959.213496:0:40303:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302272:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788876959.213537:0:40303:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:2.0:1788876959.213609:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213631:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213639:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213643:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213648:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213652:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213656:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213669:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213674:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213677:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213681:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213685:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213688:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213791:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213799:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213805:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213808:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213812:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213816:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213829:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213833:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213837:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213841:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213844:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213847:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213851:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213854:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213857:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213861:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213864:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213868:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213871:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213875:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213878:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213881:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213884:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213888:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213891:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213894:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213897:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213901:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213904:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213908:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213911:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213914:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213917:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213920:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213923:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213926:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213930:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213933:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213936:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213940:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213944:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213948:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213951:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213955:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213958:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213962:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213965:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213969:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213972:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213976:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213979:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213982:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213986:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213990:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.213993:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.213997:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214001:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214005:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214009:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214013:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214016:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214020:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214023:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214027:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214031:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214035:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214039:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214043:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214046:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214050:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214054:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214058:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214063:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214067:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214070:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214074:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214077:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214082:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214086:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214090:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214094:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214098:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214102:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214106:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214110:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214114:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214117:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214121:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214124:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214129:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214132:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214135:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214139:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214143:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214146:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214150:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214153:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214156:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214160:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214163:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214167:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214171:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214175:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214178:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214182:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214185:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214189:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214192:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214196:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214200:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214203:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214207:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00800000:00000080:2.0:1788876959.214210:0:40303:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.214214:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.214220:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00020000:00000004:2.0:1788876959.214229:0:40303:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1898, max_age 1950 00000100:00100000:2.0:1788876959.214277:0:40303:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302400:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876959.215372:0:40303:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64480 going to sleep for 11 seconds 00000100:00100000:2.0:1788876959.236995:0:40303:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302400:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876959.237042:0:40303:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788876959.237224:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.237249:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00020000:00000004:2.0:1788876959.237259:0:40303:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1898, max_age 1950 00000100:00100000:2.0:1788876959.237305:0:40303:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302528:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788876959.237422:0:40303:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06184c64480 going to sleep for 11 seconds 00000100:00100000:2.0:1788876959.242640:0:40303:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40303:1875771511302528:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788876959.242687:0:40303:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788876959.245117:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245149:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245163:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245169:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245173:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245177:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245181:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245185:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245190:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245193:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245197:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245200:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245204:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245208:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245212:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245216:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245220:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245223:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245227:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245230:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245234:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245237:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245242:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245246:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245251:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245255:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245259:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245262:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245267:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245271:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245275:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245278:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245282:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245287:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245290:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245293:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245297:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245300:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245304:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245307:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245311:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245314:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245325:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245328:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245332:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245335:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245339:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245342:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245347:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245351:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245355:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245359:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245363:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245367:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245371:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245374:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245378:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245402:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245408:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245412:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245416:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245420:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245424:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245429:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245433:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245436:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245439:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245443:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245447:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245451:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245454:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245458:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245461:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245464:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245469:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245472:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245487:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245492:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245496:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245500:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245504:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245507:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245511:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245514:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245519:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245522:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245526:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245538:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245542:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245546:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245550:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245554:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245557:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245560:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245564:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245566:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245570:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245573:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245577:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245582:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245585:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245588:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245592:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245596:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245599:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245602:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245606:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245609:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245612:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245615:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245617:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245620:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245623:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245626:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245630:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245632:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245636:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245639:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245642:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245645:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245649:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245651:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245655:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245657:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788876959.245843:0:40303:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc72b6e5d0 00020000:00000080:2.0:1788876959.245849:0:40303:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200008:2.0:1788876959.245867:0:40303:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00010000:00010000:3.0:1788876959.491357:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876959.873151:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788876959.873175:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511302656:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876960.509832:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876961.533638:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876962.557749:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876963.582456:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876965.000667:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511302784:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876965.000693:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511303040:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876965.000706:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511303296:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876965.000716:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511302912:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876965.000729:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511303168:192.168.203.111@tcp:400:kworker.0 00000001:02000400:3.0:1788876966.138875:0:40584:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg311-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 00010000:00010000:1.0:1788876966.655286:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1788876968.550562:0:40732:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:1.0:1788876968.701595:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876969.725848:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876970.111500:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511303552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.111620:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511303424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.111741:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511303680:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.111781:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511303936:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876970.111953:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511303808:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.122473:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511303424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.122498:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511303680:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.122509:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511303936:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.124528:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511303552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788876970.124551:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511303808:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788876970.750633:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788876970.984095:0:40771:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe680 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40771:1875771511304064:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876970.984139:0:40771:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023705/1024000 00000080:00200080:3.0:1788876970.984229:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.984252:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00800000:00000004:3.0:1788876970.984261:0:40771:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1951, max_age 1961 00000100:00100000:3.0:1788876970.984307:0:40771:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40771:1875771511304192:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788876970.984433:0:40771:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76e40 going to sleep for 11 seconds 00000100:00100000:3.0:1788876970.994265:0:40771:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40771:1875771511304192:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788876970.994308:0:40771:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:3.0:1788876970.994441:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994464:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994474:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994479:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994483:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994487:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994491:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994495:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994499:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994503:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994508:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994511:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994515:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994519:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994525:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994528:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994532:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994535:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994539:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994542:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994546:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994550:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994554:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994557:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994560:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994564:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994567:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994570:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994574:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994577:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994580:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994584:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994587:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994590:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994595:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994598:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994602:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994605:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994609:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994613:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994617:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994620:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994624:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994627:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994630:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994634:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994639:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994642:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994646:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994649:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994653:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994656:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994660:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994663:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994666:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994670:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994674:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994677:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994681:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994685:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994689:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994692:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994696:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994699:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994703:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994706:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994709:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994712:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994716:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994719:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994723:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994727:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994730:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994734:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994738:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994741:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994745:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994748:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994751:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994755:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994758:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994762:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994779:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994784:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994787:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994791:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994795:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994798:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994801:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994804:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994808:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994812:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994815:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994818:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994822:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994825:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994830:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994833:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994836:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994840:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994844:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994847:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994851:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994854:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994858:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994862:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994866:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994869:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994873:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994877:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994881:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994885:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994889:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994892:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994896:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994900:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994904:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994907:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994910:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994914:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994919:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994923:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994926:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994930:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994933:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00800000:00000080:3.0:1788876970.994936:0:40771:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876970.994940:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876970.994947:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00020000:00000004:3.0:1788876970.994956:0:40771:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1951, max_age 1961 00000100:00100000:3.0:1788876970.995007:0:40771:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40771:1875771511304320:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876970.998285:0:40771:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76e40 going to sleep for 11 seconds 00000100:00100000:3.0:1788876971.008592:0:40771:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40771:1875771511304320:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876971.008644:0:40771:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788876971.008725:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.008749:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00020000:00000004:3.0:1788876971.008759:0:40771:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1951, max_age 1961 00000100:00100000:3.0:1788876971.008809:0:40771:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40771:1875771511304448:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788876971.008906:0:40771:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76e40 going to sleep for 11 seconds 00000100:00100000:3.0:1788876971.014000:0:40771:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:40771:1875771511304448:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788876971.014044:0:40771:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788876971.014125:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014149:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014157:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014161:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014166:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014171:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014175:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014178:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014183:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014187:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014190:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014194:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014198:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014201:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014205:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014209:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014213:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014216:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014220:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014223:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014227:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014231:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014235:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014238:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014242:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014246:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014250:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014253:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014258:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014262:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014267:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014270:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014274:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014278:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014282:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014286:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014290:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014294:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014298:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014302:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014305:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014309:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014313:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014316:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014320:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014324:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014327:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014331:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014335:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014339:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014342:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014346:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014351:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014354:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014359:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014363:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014367:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.014371:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.014376:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015111:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015129:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015138:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015142:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015146:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015150:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015154:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015158:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015161:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015165:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015169:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015173:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015177:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015181:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015184:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015189:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015192:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015196:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015201:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015205:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015209:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015214:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015218:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015223:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015227:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015231:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015235:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015240:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015243:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015248:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015252:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015256:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015260:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015264:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015269:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015274:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015278:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015282:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015286:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015290:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015294:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015299:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015302:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015307:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015311:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015315:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015319:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015324:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015328:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015332:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015336:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015340:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015344:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015348:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015351:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015355:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015359:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015362:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015366:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015370:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015373:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015377:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015400:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015406:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015410:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200080:3.0:1788876971.015414:0:40771:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffe7c243190 00020000:00000080:3.0:1788876971.015417:0:40771:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838ac00 uarg=0000000000000000 00000080:00200008:3.0:1788876971.015435:0:40771:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:3.0:1788876971.098171:0:40788:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876971.098214:0:40788:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876971.098241:0:40788:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876971.098262:0:40788:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788876971.098272:0:40788:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876971.098282:0:40788:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876971.098286:0:40788:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:3.0:1788876971.098295:0:40788:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac1c2fb58 00010000:00010000:3.0:1788876971.098299:0:40788:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788876971.098309:0:40788:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1788876971.098328:0:40788:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876971.098342:0:40788:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=9 00000080:00200000:3.0:1788876971.098347:0:40788:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788876971.098361:0:40788:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f9.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788876971.098447:0:40788:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 40788 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.098456:0:40788:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 40788 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.098465:0:40788:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 40788 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788876971.098497:0:40788:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:40788:1875771511304576:192.168.203.111@tcp:101:ls.0 00000100:00100000:3.0:1788876971.098624:0:40788:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76e40 going to sleep for 11 seconds 00000100:00100000:3.0:1788876971.113516:0:40788:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ls:f0204769-56bd-4f5c-ab97-74c450c9e072:40788:1875771511304576:192.168.203.111@tcp:101:ls.0 00010000:00010000:3.0:1788876971.113559:0:40788:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113590:0:40788:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113600:0:40788:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113615:0:40788:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113630:0:40788:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113645:0:40788:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113654:0:40788:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113663:0:40788:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788876971.113676:0:40788:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188873100 x1875771511304576/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788876987 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'ls.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788876971.113691:0:40788:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113703:0:40788:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/274 (0 0) 00000080:00200000:3.0:1788876971.113711:0:40788:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788876971.113740:0:40788:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x112:0x0], updating i_size 0 00000080:00200000:3.0:1788876971.113744:0:40788:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x112:0x0](ffffa06198289148): rc = 0 00000080:00200000:3.0:1788876971.113775:0:40788:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x112:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:3.0:1788876971.113782:0:40788:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) for lock 0x7d14dcdc37048a00 00000080:00010000:3.0:1788876971.113799:0:40788:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876971.113804:0:40788:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a00 from it ffffb95ac1c2fb20 00010000:00010000:3.0:1788876971.113807:0:40788:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788876971.113817:0:40788:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788876971.113872:0:40788:0:(file.c:6296:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffffa06198289148), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1788876971.113880:0:40788:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffffa06198289148),name=f9.replay-single 00000002:00010000:3.0:1788876971.113910:0:40788:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876971.113921:0:40788:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876971.113934:0:40788:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788876971.113941:0:40788:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876971.113948:0:40788:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876971.113951:0:40788:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) for lock 0x7d14dcdc37048a00 00000080:00010000:3.0:1788876971.113955:0:40788:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a00 from it ffffb95ac1c2fcb8 00010000:00010000:3.0:1788876971.113958:0:40788:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788876971.113966:0:40788:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876971.114028:0:40788:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788876971.114036:0:40788:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876971.114044:0:40788:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1788876971.114052:0:40788:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1788876971.114059:0:40788:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876971.114063:0:40788:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788876971.114066:0:40788:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:3.0:1788876971.114069:0:40788:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac1c2fb58 00010000:00010000:3.0:1788876971.114073:0:40788:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:3.0:1788876971.114080:0:40788:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:3.0:1788876971.114089:0:40788:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788876971.114099:0:40788:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=8 00000080:00200008:3.0:1788876971.114105:0:40788:0:(file.c:6296:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffffa06198289148), request_mask 258, flags 0, foreign 0 00000080:00200000:3.0:1788876971.114108:0:40788:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffffa06198289148),name=f9.replay-single 00000002:00010000:3.0:1788876971.114116:0:40788:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788876971.114122:0:40788:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876971.114131:0:40788:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788876971.114138:0:40788:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788876971.114142:0:40788:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00010000:3.0:1788876971.114144:0:40788:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) for lock 0x7d14dcdc37048a00 00000080:00010000:3.0:1788876971.114148:0:40788:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a00 from it ffffb95ac1c2fcb8 00010000:00010000:3.0:1788876971.114151:0:40788:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788876971.114158:0:40788:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876971.152900:0:40791:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876971.152941:0:40791:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876971.152966:0:40791:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1788876971.152992:0:40791:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:1.0:1788876971.153001:0:40791:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876971.153012:0:40791:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876971.153017:0:40791:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:1.0:1788876971.153027:0:40791:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac1c2fb58 00010000:00010000:1.0:1788876971.153032:0:40791:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:1.0:1788876971.153044:0:40791:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1788876971.153064:0:40791:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876971.153074:0:40791:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=8 00000080:00200008:1.0:1788876971.153098:0:40791:0:(file.c:6296:ll_getattr_dentry()) START file f9.replay-single:[0x200001b71:0x112:0x0](ffffa06198289148), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788876971.153104:0:40791:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x112:0x0](ffffa06198289148),name=f9.replay-single 00000002:00010000:1.0:1788876971.153121:0:40791:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x112:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876971.153129:0:40791:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876971.153140:0:40791:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788876971.153148:0:40791:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876971.153152:0:40791:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) 00000080:00010000:1.0:1788876971.153156:0:40791:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x112:0x0](ffffa06198289148) for lock 0x7d14dcdc37048a00 00000080:00010000:1.0:1788876971.153160:0:40791:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a00 from it ffffb95ac1c2fcb8 00010000:00010000:1.0:1788876971.153163:0:40791:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788876971.153172:0:40791:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1788876971.153208:0:40791:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876971.153215:0:40791:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876971.153222:0:40791:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1788876971.153231:0:40791:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1788876971.153239:0:40791:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876971.153243:0:40791:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876971.153246:0:40791:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:1.0:1788876971.153249:0:40791:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac1c2fc10 00010000:00010000:1.0:1788876971.153253:0:40791:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1788876971.153262:0:40791:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1788876971.153272:0:40791:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788876971.153280:0:40791:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f9.replay-single, flags=0 00000080:00200000:1.0:1788876971.153285:0:40791:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788876971.153291:0:40791:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788876971.153298:0:40791:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1788876971.153307:0:40791:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00000080:00200000:1.0:1788876971.153315:0:40791:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788876971.153319:0:40791:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788876971.153322:0:40791:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc370489f9 00000080:00010000:1.0:1788876971.153326:0:40791:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc370489f9 from it ffffb95ac1c2fd68 00010000:00010000:1.0:1788876971.153329:0:40791:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 1 00010000:00010000:1.0:1788876971.153338:0:40791:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000080:00200000:1.0:1788876971.153347:0:40791:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788876971.153353:0:40791:0:(namei.c:2351:ll_unlink()) VFS Op:name=f9.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:1.0:1788876971.153412:0:40791:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1788876971.153444:0:40791:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:1.0:1788876971.153457:0:40791:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x112:0x0]: losing layout lock 00010000:00010000:1.0:1788876971.153640:0:40791:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876971.153656:0:40791:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00010000:00010000:1.0:1788876971.153664:0:40791:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:1.0:1788876971.153668:0:40791:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ac00/0x7d14dcdc37048a00 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x112:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313b03a9 expref: -99 pid: 40788 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:1.0:1788876971.153692:0:40791:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a800/0x7d14dcdc370489f9 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b038d expref: -99 pid: 40303 timeout: 0 lvb_type: 0 lru_score: 11 lru_type: 1 00000020:00100000:1.0:1788876971.153718:0:40791:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788876971.153736:0:40791:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188699c00 x1875771511304832/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 504/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788876971.153777:0:40791:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:40791:1875771511304832:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788876971.154068:0:40791:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a1ed6840 going to sleep for 11 seconds 00000100:00100000:3.0:1788876971.163754:0:40791:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:40791:1875771511304832:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788876971.163913:0:40791:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x112:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788876971.163926:0:40791:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x112:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788876971.163935:0:40791:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x112:0x0](ffffa06198289148) 00010000:00010000:1.0:1788876971.777303:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876972.799958:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876974.845609:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876975.246034:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305088:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.246153:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305344:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.246190:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1788876975.246197:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511304960:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.246307:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305216:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.246334:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305472:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.266703:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305088:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.266730:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305344:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.266778:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511304960:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.266792:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305216:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876975.266887:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511305472:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876975.870519:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876977.921270:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876978.944833:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876979.966626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1788876979.995261:0:41366:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 10: create |X| rename unlink ======= 10:16:18 (1788876978) 00000080:00200000:2.0:1788876980.165048:0:41374:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788876980.165069:0:41374:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a07 00000080:00010000:2.0:1788876980.165079:0:41374:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876980.165082:0:41374:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a07 from it ffffb95ac1cefbd0 00010000:00010000:2.0:1788876980.165089:0:41374:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788876980.165103:0:41374:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876980.165140:0:41374:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876980.165171:0:41374:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:2.0:1788876980.165176:0:41374:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876980.165192:0:41374:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876980.165205:0:41374:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876980.165216:0:41374:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788876980.165224:0:41374:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876980.165231:0:41374:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876980.165234:0:41374:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a07 00000080:00010000:2.0:1788876980.165237:0:41374:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a07 from it ffffb95ac1cefcd0 00010000:00010000:2.0:1788876980.165241:0:41374:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788876980.165249:0:41374:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876980.165257:0:41374:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876980.165263:0:41374:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876980.165269:0:41374:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876980.165274:0:41374:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876980.165287:0:41374:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788876980.165293:0:41374:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876980.165297:0:41374:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876980.165299:0:41374:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a07 00000080:00010000:2.0:1788876980.165302:0:41374:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a07 from it ffffb95ac1cefdb0 00010000:00010000:2.0:1788876980.165305:0:41374:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788876980.165312:0:41374:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876980.165320:0:41374:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876980.165325:0:41374:0:(namei.c:2062:ll_create_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:2.0:1788876980.165329:0:41374:0:(namei.c:2017:ll_mknod()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:2.0:1788876980.165354:0:41374:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876980.165522:0:41374:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876980.165531:0:41374:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876980.165534:0:41374:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a07 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b03d3 expref: -99 pid: 41374 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1788876980.165616:0:41374:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876980.165629:0:41374:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888af480 x1875771511305856/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876980.165655:0:41374:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:41374:1875771511305856:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876980.165748:0:41374:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c1780 going to sleep for 11 seconds 00000100:00100000:2.0:1788876980.186707:0:41374:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:41374:1875771511305856:192.168.203.111@tcp:36:mcreate.0 00010000:00010000:1.0:1788876980.989605:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1788876981.190842:0:41394:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876981.190870:0:41394:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876981.190888:0:41394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876981.190902:0:41394:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788876981.190909:0:41394:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876981.190916:0:41394:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876981.190920:0:41394:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a0e 00000080:00010000:2.0:1788876981.190927:0:41394:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a0e from it ffffb95ac1dcfbd0 00010000:00010000:2.0:1788876981.190931:0:41394:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788876981.190941:0:41394:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876981.190957:0:41394:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788876981.190972:0:41394:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:2.0:1788876981.190976:0:41394:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876981.190982:0:41394:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876981.190988:0:41394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876981.190996:0:41394:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788876981.191002:0:41394:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876981.191006:0:41394:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876981.191008:0:41394:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a0e 00000080:00010000:2.0:1788876981.191011:0:41394:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a0e from it ffffb95ac1dcfcd0 00010000:00010000:2.0:1788876981.191014:0:41394:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788876981.191021:0:41394:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876981.191028:0:41394:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876981.191033:0:41394:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788876981.191039:0:41394:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788876981.191044:0:41394:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876981.191052:0:41394:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788876981.191058:0:41394:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788876981.191061:0:41394:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788876981.191064:0:41394:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a0e 00000080:00010000:2.0:1788876981.191067:0:41394:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a0e from it ffffb95ac1dcfdb0 00010000:00010000:2.0:1788876981.191070:0:41394:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788876981.191077:0:41394:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:2.0:1788876981.191083:0:41394:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788876981.191087:0:41394:0:(namei.c:2062:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:2.0:1788876981.191091:0:41394:0:(namei.c:2017:ll_mknod()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:2.0:1788876981.191115:0:41394:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876981.191158:0:41394:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:2.0:1788876981.191165:0:41394:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1788876981.191168:0:41394:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cfc00/0x7d14dcdc37048a0e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b03ef expref: -99 pid: 41387 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:2.0:1788876981.191190:0:41394:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788876981.191202:0:41394:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888adf80 x1875771511307264/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788876981.191223:0:41394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:41394:1875771511307264:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:2.0:1788876981.193569:0:41394:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c1780 going to sleep for 11 seconds 00000100:00100000:2.0:1788876981.210363:0:41394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:41394:1875771511307264:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:2.0:1788876981.210547:0:41394:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x114:0x0](ffffa0619828e3c8) 00000080:00200000:2.0:1788876981.210674:0:41394:0:(namei.c:2070:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, unhashed 1 00010000:00010000:3.0:1788876982.014477:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876983.040063:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876984.061764:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876985.085782:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876986.109709:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876987.134073:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876988.158979:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876989.182415:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876990.206004:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876990.589814:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511309568:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876990.589913:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511309824:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876990.589945:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:1.0:1788876990.589952:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511309440:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876990.589978:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511309696:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876990.589999:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511309952:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788876991.229642:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876992.253688:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876993.278882:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876994.301629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876995.326113:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788876995.710437:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511310080:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876995.710515:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511310336:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876995.710548:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511310592:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876995.715171:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511310592:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788876995.725555:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511310336:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788876995.725926:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511310720:192.168.203.111@tcp:38:ptlrpcd_00_01.0 00000100:00080000:3.0:1788876995.746116:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788876995.746130:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788876995.746135:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511310720:192.168.203.111@tcp:38:ptlrpcd_00_01.0 00010000:00010000:1.0:1788876996.349596:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788876997.373547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788876998.399162:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788876999.421625:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877000.447866:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788877000.829843:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511310976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877000.830606:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511310848:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877000.830678:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311232:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877000.830703:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1788877000.830709:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311104:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877000.830733:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311360:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788877000.833976:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877000.833989:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788877000.833994:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511310976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877000.834097:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311360:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877000.834110:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311232:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877000.835815:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311104:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788877001.469894:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877002.496943:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877003.517527:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877004.542254:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877005.565582:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788877005.950145:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511311616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877005.950918:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511311488:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877005.950975:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311872:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877005.951007:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1788877005.951014:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311744:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877005.951040:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511312000:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788877005.976281:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877005.976300:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:3.0:1788877005.976306:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511311616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877005.985366:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511311872:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877005.985474:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511312000:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877006.590718:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877007.616657:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877008.637647:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877009.661697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877010.685752:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877011.081320:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877011.081331:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877011.081335:0:8:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877011.081349:0:8:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1997 00000100:00080000:3.0:1788877011.081364:0:8:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1788877011.081456:0:8:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877011.081474:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877011.082040:0:8:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511312256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877011.082126:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511312128:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.085328:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511312512:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.085480:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1788877011.085498:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511312384:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.085526:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511312640:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.085568:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511312256:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788877011.126652:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877011.126676:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788877011.126685:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511312256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877011.127169:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511312384:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.127185:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511312640:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.127198:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511312512:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877011.711530:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1788877011.837696:0:2395:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788876995/real 1788876995] req@ffffa06198afd880 x1875771511310080/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877011 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877011.837742:0:2395:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06198afd880 x1875771511310080/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877011 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877011.837752:0:2395:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1788877011.837756:0:2395:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.111@tcp: Connection to MGS (at 192.168.203.111@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1788877011.837760:0:2395:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.111@tcp for MGC192.168.203.111@tcp not replayable, auto-deactivating 00000100:00080000:1.0:1788877011.837762:0:2395:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:1.0:1788877011.837766:0:2395:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06198afd880 x1875771511310080/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877011 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877011.837775:0:2395:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06198afed80 x1875771511310848/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877016 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877011.837801:0:2395:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06198affb80 x1875771511311488/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877021 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877011.837811:0:2395:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188873100 x1875771511312128/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877027 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:1.0:1788877011.837838:0:2395:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1788877011.837841:0:2395:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:1.0:1788877011.837842:0:2395:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:1.0:1788877011.837845:0:2395:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:1.0:1788877011.837937:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511310080:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.838426:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511312128:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788877011.842550:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877011.842565:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877011.842578:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877011.842589:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 1939 00000100:00080000:2.0:1788877011.842611:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877011.842668:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511312768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877011.842778:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511312768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877011.842926:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877011.842930:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877011.842932:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877011.842936:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2002 00000100:00080000:2.0:1788877011.842940:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877011.842945:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877011.842957:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877011.842972:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511312896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877011.843065:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.843150:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511313024:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877011.843238:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511312896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877011.848001:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511313280:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877011.852159:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313152:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877011.863583:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877011.863602:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788877011.863609:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511312896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877011.863727:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511313024:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877011.863743:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511313280:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877012.733607:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877013.777746:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877014.781696:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877015.811032:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877016.830850:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877017.213489:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877017.213499:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877017.213502:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877017.213505:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877017.213509:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877017.213520:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2003 00000100:00080000:2.0:1788877017.213531:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877017.213538:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877017.213559:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877017.213606:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511313408/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877017.213754:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313536:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877017.213836:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313792:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877017.213862:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788877017.213868:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313664:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877017.214309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511313408:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877017.218905:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313536:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877017.221099:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313792:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877017.227719:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511313664:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877017.228715:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877017.228725:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788877017.228730:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511313408:192.168.203.111@tcp:38:kworker.0 00010000:00010000:1.0:1788877017.853626:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877018.877582:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877019.901790:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877020.925764:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788877021.309691:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877011/real 1788877011] req@ffffa061888adc00 x1875771511312768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877021 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877021.309726:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511312768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877021 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877021.309744:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877021.309765:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877021.309769:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511312768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877021.310055:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511313920:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877021.310134:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511314048:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877021.310505:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511314176:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877021.310560:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511314432:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877021.310585:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788877021.310591:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511314304:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788877021.327742:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) MGC192.168.203.111@tcp: connect to target with instance 0 10000000:01000000:3.0:1788877021.327759:0:2394:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:02000400:3.0:1788877021.327767:0:2394:0:(import.c:1328:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.203.111@tcp) after server handle changed from 0xf5bf0f7e313aff73 to 0xf5bf0f7e313b0603 00000100:00080000:3.0:1788877021.327772:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:3.0:1788877021.327778:0:2394:0:(import.c:1718:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.203.111@tcp; invalidating 00010000:00010000:1.0:1788877021.328077:0:2396:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1788877021.328088:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511314304:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877021.328109:0:2396:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 2 blwis in pool 00000100:00100000:1.0:1788877021.328114:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511314176:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877021.328128:0:2396:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 3 blwis in pool 00000100:00100000:1.0:1788877021.328132:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511314432:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877021.328162:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (2 blwis in pool) 00010000:00010000:1.0:1788877021.328173:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (1 blwis in pool) 00010000:00010000:1.0:1788877021.328179:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788877021.328183:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877021.328184:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877021.328186:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877021.328242:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877021.328524:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511313920:192.168.203.111@tcp:250:kworker.0 00010000:00080000:1.0:1788877021.328551:0:2394:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa06198affb80 x1875771511314048/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/416 e 0 to 0 dl 1788877046 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877021.328564:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa06189393000: connect to target with instance 22 00000100:00080000:1.0:1788877021.328580:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:1.0:1788877021.328585:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1788877021.328588:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 90194313218 gen 1 00000100:00100000:1.0:1788877021.328594:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888adf80 x1875771511307264/t90194313219(90194313219) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876997 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877021.328601:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 90194313218 last 0 00000100:00080000:1.0:1788877021.328607:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa061888adf80 x1875771511307264/t90194313219(90194313219) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788876997 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877021.328619:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511314048:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877021.328639:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511307264:192.168.203.111@tcp:36:mcreate.0 00000100:00080000:3.0:1788877021.329104:0:41697:0:(import.c:1642:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.203.111@tcp to MGS@192.168.203.111@tcp 00000100:00100000:3.0:1788877021.329109:0:41697:0:(import.c:323:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:3.0:1788877021.329112:0:41697:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808003 00010000:00010000:3.0:1788877021.329125:0:41697:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b00a0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788877021.329135:0:41697:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b00a0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788877021.329140:0:41697:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1788877021.329157:0:41697:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788877021.329164:0:41697:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06180030600/0x7d14dcdc370489e4 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b00a0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877021.329177:0:41697:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886cf000/0x7d14dcdc370489f2 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b00ae expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788877021.329183:0:41697:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886cf000/0x7d14dcdc370489f2 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b00ae expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788877021.329188:0:41697:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1788877021.329191:0:41697:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788877021.329194:0:41697:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886cf000/0x7d14dcdc370489f2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b00ae expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877021.329207:0:41697:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886cea00/0x7d14dcdc370489dd lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b0099 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788877021.329213:0:41697:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886cea00/0x7d14dcdc370489dd lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b0099 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788877021.329222:0:41697:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:3.0:1788877021.329225:0:41697:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788877021.329228:0:41697:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886cea00/0x7d14dcdc370489dd lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b0099 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877021.329247:0:41697:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa061886cf200/0x7d14dcdc370489eb lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b00a7 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1788877021.329255:0:41697:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa061886cf200/0x7d14dcdc370489eb lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b00a7 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788877021.329259:0:41697:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1788877021.329263:0:41697:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788877021.329266:0:41697:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa061886cf200/0x7d14dcdc370489eb lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b00a7 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:3.0:1788877021.329280:0:41697:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:3.0:1788877021.329284:0:41697:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from RECOVER to FULL 10000000:01000000:3.0:1788877021.329286:0:41697:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1788877021.329289:0:41697:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:3.0:1788877021.329292:0:41697:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) MGC192.168.203.111@tcp: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00010000:00010000:1.0:1788877021.949745:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877022.979699:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:3.0:1788877023.167090:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1788877023.167097:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffffa06189393000 from 88 10000000:01000000:3.0:1788877023.167103:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1788877023.167149:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877023.167161:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877023.167178:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877023.167205:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314560:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788877023.167308:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877023.170902:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314560:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788877023.170931:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06ce expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877023.170946:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b06ce expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877023.171024:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314688:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1788877023.171107:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877023.175066:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314688:192.168.203.111@tcp:256:ll_cfg_requeue.0 10000000:01000000:3.0:1788877023.175154:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1788877023.175164:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06ce expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877023.175176:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06ce expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788877023.175185:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1788877023.175188:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06189393000 from 52 10000000:01000000:3.0:1788877023.175190:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1788877023.175222:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877023.175241:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877023.175252:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877023.175271:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314816:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788877023.175593:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877023.189299:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314816:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788877023.189329:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06d5 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877023.189341:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b06d5 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877023.189462:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314944:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788877023.189537:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877023.202076:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511314944:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788877023.202135:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315072:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1788877023.202223:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187d76780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877023.204830:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315072:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1788877023.206124:0:41734:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 64768 count 51 00000020:01000000:1.0:1788877023.206687:0:15194:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 52-51 (rc=0) 10000000:01000000:1.0:1788877023.206707:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:1.0:1788877023.206718:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06d5 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877023.206729:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06d5 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1788877023.206739:0:15194:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:1.0:1788877023.206741:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06189393000 from 7 10000000:01000000:1.0:1788877023.206744:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:1.0:1788877023.206782:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877023.206791:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877023.206801:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788877023.206826:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315200:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1788877023.206904:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a1f09540 going to sleep for 11 seconds 00000100:00100000:1.0:1788877023.209297:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315200:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1788877023.209325:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06dc expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877023.209343:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b06dc expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788877023.209457:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315328:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1788877023.209508:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a1f09540 going to sleep for 11 seconds 00000100:00100000:1.0:1788877023.219959:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315328:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1788877023.220020:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315456:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:1.0:1788877023.220140:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a1f09540 going to sleep for 11 seconds 00000100:00100000:1.0:1788877023.225148:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315456:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:3.0:1788877023.226184:0:41735:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00000020:01000000:1.0:1788877023.231215:0:15194:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 7-6 (rc=0) 10000000:01000000:1.0:1788877023.231256:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:1.0:1788877023.231268:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06dc expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877023.231281:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06dc expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788877023.231582:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:2.0:1788877023.231587:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffffa0618906b380 from 1 10000000:01000000:2.0:1788877023.231591:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1788877023.231635:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877023.231645:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877023.231657:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788877023.231686:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315584:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1788877023.231778:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c13c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877023.234364:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315584:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1788877023.234426:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06e3 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877023.234440:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b06e3 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788877023.234507:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315712:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1788877023.234572:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c13c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877023.240827:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511315712:192.168.203.111@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1788877023.240875:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1788877023.240885:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06e3 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877023.240895:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b06e3 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877025.023833:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877026.047417:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877026.310541:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa061888adf80 x1875771511307264/t90194313219(90194313219) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788877037 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.310555:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1788877026.310558:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 90194313218 gen 1 00000100:00100000:1.0:1788877026.310572:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa061888adf80 x1875771511307264/t90194313219(90194313219) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788877037 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.310587:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 90194313218 last 90194313219 00000100:00080000:1.0:1788877026.310595:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa06198aff480 x1875771511307776/t90194313220(90194313220) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788876997 ref 1 fl New:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877026.310606:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511307264:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:1.0:1788877026.310625:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511307776:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788877026.314119:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 90194313219 gen 1 00000100:00100000:1.0:1788877026.314146:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06198aff480 x1875771511307776/t90194313220(90194313220) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788877042 ref 2 fl Rpc:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.314160:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa06198aff480 x1875771511307776/t90194313220(90194313220) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788877042 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.314169:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1788877026.314172:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 90194313219 gen 1 00000100:00100000:1.0:1788877026.314176:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06198aff480 x1875771511307776/t90194313220(90194313220) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788877042 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.314184:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 90194313219 last 90194313220 00000100:00080000:1.0:1788877026.314191:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa06198afdc00 x1875771511309056/t90194313221(90194313221) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:13/10 lens 536/456 e 0 to 0 dl 1788877004 ref 1 fl New:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877026.314203:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511307776:192.168.203.111@tcp:36:rm.0 00000100:00100000:1.0:1788877026.314219:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511309056:192.168.203.111@tcp:36:mv.0 00000100:00080000:1.0:1788877026.335728:0:2394:0:(client.c:3439:ptlrpc_replay_interpret()) @@@ got reply req@ffffa06198afdc00 x1875771511309056/t90194313221(90194313221) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:13/10 lens 536/456 e 0 to 0 dl 1788877042 ref 2 fl Interpret:RQU/204/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.335749:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:1.0:1788877026.335751:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 90194313219 gen 1 00000100:00100000:1.0:1788877026.335758:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06198aff480 x1875771511307776/t90194313220(90194313220) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/456 e 0 to 0 dl 1788877042 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.335766:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 90194313219 last 90194313221 00000100:00080000:1.0:1788877026.335771:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY to REPLAY_LOCKS 00000100:00100000:1.0:1788877026.336453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511309056:192.168.203.111@tcp:36:mv.0 00010000:00080000:3.0:1788877026.336521:0:41746:0:(ldlm_request.c:2972:ldlm_lock_replay_thread()) lock replay thread lustre-MDT0000-mdc-ffffa06189393000 to lustre-MDT0000_UUID@192.168.203.111@tcp 00010000:00010000:3.0:1788877026.336526:0:41746:0:(ldlm_request.c:2888:ldlm_cancel_unused_locks_for_replay()) Dropping as many unused locks as possible before replay for namespace lustre-MDT0000-mdc-ffffa06189393000 (1) 00010000:00010000:3.0:1788877026.336581:0:41746:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce800/0x7d14dcdc37048a38 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000000 nid: local remote: 0xf5bf0f7e313b04ac expref: -99 pid: 41465 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877026.336598:0:41746:0:(ldlm_request.c:2901:ldlm_cancel_unused_locks_for_replay()) Canceled 0 unused locks from namespace lustre-MDT0000-mdc-ffffa06189393000 00000100:00080000:3.0:1788877026.336989:0:41746:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_LOCKS to REPLAY_WAIT 00000100:00100000:1.0:1788877026.337310:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511315840:192.168.203.111@tcp:400:ldlm_lock_repla.0 00000100:00100000:1.0:1788877026.361891:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 90194313221 gen 1 00000100:00080000:1.0:1788877026.361920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY_WAIT to RECOVER 00000100:00080000:1.0:1788877026.361932:0:2394:0:(client.c:3248:ptlrpc_resend_req()) @@@ going to resend req@ffffa06188873100 x1875771511315840/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788877042 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.361942:0:2394:0:(client.c:3257:ptlrpc_resend_req()) @@@ it has reply, so skip it req@ffffa06188873100 x1875771511315840/t0(0) o400->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 0 dl 1788877042 ref 1 fl Interpret:RQU/2c0/0 rc 0/0 job:'ldlm_lock_repla.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877026.361950:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from RECOVER to FULL 00000100:02000000:1.0:1788877026.362373:0:2394:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) lustre-MDT0000-mdc-ffffa06189393000: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00000100:00100000:1.0:1788877026.362379:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511315840:192.168.203.111@tcp:400:ldlm_lock_repla.0 00000100:00100000:2.0:1788877026.435019:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511315968:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877026.435120:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511316224:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877026.435151:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788877026.435157:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511316096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877026.435181:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511316352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877026.454096:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511316096:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877026.454121:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511316352:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877026.454137:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511315968:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877026.454152:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511316224:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788877027.070893:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877028.095503:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877029.119316:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877030.141690:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788877030.160103:0:41790:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316480:192.168.203.111@tcp:101:lfs.0 00010000:00010000:2.0:1788877030.160142:0:41790:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160157:0:41790:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160171:0:41790:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160185:0:41790:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160593:0:41790:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160604:0:41790:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160611:0:41790:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788877030.160626:0:41790:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06198afc700 x1875771511316480/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877046 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788877030.160641:0:41790:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160656:0:41790:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788877030.160665:0:41790:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788877030.160703:0:41790:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788877030.160710:0:41790:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:2.0:1788877030.160716:0:41790:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877030.160719:0:41790:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac214fcb8 00010000:00010000:2.0:1788877030.160723:0:41790:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877030.160733:0:41790:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788877030.161718:0:41790:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1788877030.161729:0:41790:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877030.161761:0:41790:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877030.161782:0:41790:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877030.161799:0:41790:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788877030.161807:0:41790:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877030.161815:0:41790:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877030.161819:0:41790:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:2.0:1788877030.161826:0:41790:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac214fcb8 00010000:00010000:2.0:1788877030.161830:0:41790:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877030.161838:0:41790:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877030.161997:0:41790:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877030.162006:0:41790:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877030.162014:0:41790:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788877030.162024:0:41790:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877030.162030:0:41790:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877030.162034:0:41790:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877030.162036:0:41790:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:2.0:1788877030.162040:0:41790:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac214fbb0 00010000:00010000:2.0:1788877030.162043:0:41790:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788877030.162050:0:41790:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788877030.162059:0:41790:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:2.0:1788877030.162069:0:41790:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:2.0:1788877030.162176:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.162194:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00800000:00000004:2.0:1788877030.162202:0:41790:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 1972, max_age 2021 00000100:00100000:2.0:1788877030.162256:0:41790:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316608:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788877030.162366:0:41790:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c1480 going to sleep for 11 seconds 00000100:00100000:2.0:1788877030.170718:0:41790:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316608:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788877030.170756:0:41790:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023705/1024000 00000080:00200080:2.0:1788877030.170850:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.170872:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00800000:00000004:2.0:1788877030.170881:0:41790:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 1972, max_age 2021 00000100:00100000:2.0:1788877030.170922:0:41790:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316736:192.168.203.111@tcp:41:lfs.0 00000100:00100000:2.0:1788877030.171010:0:41790:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c1480 going to sleep for 11 seconds 00000100:00100000:2.0:1788877030.189865:0:41790:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316736:192.168.203.111@tcp:41:lfs.0 00800000:00000004:2.0:1788877030.189904:0:41790:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:2.0:1788877030.189991:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190012:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190021:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190026:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190031:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190035:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190039:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190043:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190047:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190050:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190054:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190057:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190062:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190065:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190070:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190073:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190079:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190082:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190086:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190089:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190094:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190097:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190101:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190105:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190109:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190113:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190118:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190121:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190125:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190128:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190132:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190136:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190140:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190144:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190148:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190152:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190156:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190159:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190163:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190166:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190170:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190174:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190179:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190182:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190186:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190190:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190194:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190198:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190202:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190206:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190210:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190214:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190218:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190221:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190225:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190229:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190233:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190237:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190241:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190244:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190248:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190252:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190256:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190260:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190264:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190268:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190273:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190277:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190281:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190285:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190289:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190293:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190297:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190301:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190306:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190309:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190314:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190317:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190322:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190325:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190329:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190333:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190337:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190341:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190345:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190349:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190353:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190357:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190362:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190365:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190369:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190374:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190378:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190890:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190904:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190909:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190913:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190917:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190921:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190925:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190929:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190932:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190937:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190940:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190944:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190948:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190952:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190955:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190960:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190963:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190967:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190971:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190975:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190979:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190983:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190986:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190991:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.190995:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.190999:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.191003:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.191007:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.191011:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.191016:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.191020:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.191024:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00800000:00000080:2.0:1788877030.191028:0:41790:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.191032:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.191039:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00020000:00000004:2.0:1788877030.191051:0:41790:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 1972, max_age 2021 00000100:00100000:2.0:1788877030.191104:0:41790:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316864:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788877030.191210:0:41790:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c1480 going to sleep for 11 seconds 00000100:00100000:2.0:1788877030.203133:0:41790:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316864:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788877030.203173:0:41790:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:2.0:1788877030.203254:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.203275:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00020000:00000004:2.0:1788877030.203284:0:41790:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 1972, max_age 2021 00000100:00100000:2.0:1788877030.203336:0:41790:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316992:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788877030.203464:0:41790:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061890c1480 going to sleep for 11 seconds 00000100:00100000:2.0:1788877030.209506:0:41790:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:41790:1875771511316992:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788877030.209548:0:41790:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788877030.209681:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209715:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209724:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209729:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209733:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209737:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209741:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209745:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209749:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209753:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209757:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209761:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209765:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209769:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209773:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209777:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209781:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209785:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209789:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209793:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209797:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209801:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209806:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209810:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209814:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209819:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209822:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209827:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209831:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209835:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209839:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209843:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209847:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209851:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209855:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209859:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209863:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209867:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209871:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209875:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209879:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209882:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209887:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209890:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209894:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209897:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209901:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209905:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209908:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209911:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209916:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209919:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209924:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209927:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209931:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209934:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209938:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209942:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209946:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209950:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209954:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209958:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209962:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209966:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209970:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209974:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209978:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209981:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209986:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209989:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.209994:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.209997:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210002:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210005:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210010:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210014:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210018:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210022:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210026:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210029:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210034:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210037:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210042:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210045:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210049:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210053:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210057:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210073:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210078:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210082:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210086:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210089:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210093:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210097:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210100:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210104:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210108:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210111:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210116:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210120:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210124:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210128:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210132:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210136:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210140:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210143:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210147:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210151:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210155:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210159:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210163:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210167:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210170:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210174:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210178:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210182:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210186:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210190:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210194:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210198:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210202:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210206:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210210:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210213:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200080:2.0:1788877030.210217:0:41790:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7fff8e13e0b0 00020000:00000080:2.0:1788877030.210222:0:41790:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06000 uarg=0000000000000000 00000080:00200008:2.0:1788877030.210239:0:41790:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000100:00100000:2.0:1788877031.549722:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317248:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877031.549879:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317504:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877031.549953:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:2.0:1788877031.549964:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511317120:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877031.550015:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317376:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877031.550052:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317632:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877031.560851:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511317120:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877031.560888:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317376:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877031.560902:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317632:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877031.560915:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317248:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877031.560925:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317504:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788877032.190142:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877033.215160:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877035.266530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1788877036.657821:0:42071:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg311-client.virtnet: executing wait_import_state_mount (FULL|IDLE) mdc.lustre-MDT0000-mdc-*.mds_server_uuid 1475 0 00000100:00100000:3.0:1788877036.681235:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511317760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.681327:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511318016:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.681356:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511318272:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.681419:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788877036.681428:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.681469:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511318144:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.696485:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511317888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.696505:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511317760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.696516:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511318016:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.696526:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511318272:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877036.708621:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511318144:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877038.333607:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1788877038.510038:0:42219:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: mdc.lustre-MDT0000-mdc-*.mds_server_uuid in FULL state after 0 sec 00010000:00010000:2.0:1788877040.386022:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1788877040.445377:0:42258:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788877040.445417:0:42258:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877040.449575:0:42258:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877040.449600:0:42258:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877040.449615:0:42258:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788877040.449624:0:42258:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877040.449634:0:42258:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877040.449638:0:42258:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:3.0:1788877040.449646:0:42258:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac1463cb8 00010000:00010000:3.0:1788877040.449650:0:42258:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877040.449659:0:42258:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200008:3.0:1788877040.450272:0:42258:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1788877040.450278:0:42258:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877040.450289:0:42258:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877040.450296:0:42258:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788877040.450305:0:42258:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788877040.450311:0:42258:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877040.450316:0:42258:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877040.450318:0:42258:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:3.0:1788877040.450322:0:42258:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac1463cb8 00010000:00010000:3.0:1788877040.450326:0:42258:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788877040.450333:0:42258:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788877040.450758:0:42258:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877040.450767:0:42258:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877040.450775:0:42258:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788877040.450783:0:42258:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788877040.450794:0:42258:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877040.450798:0:42258:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877040.450801:0:42258:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:3.0:1788877040.450805:0:42258:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac1463bb0 00010000:00010000:3.0:1788877040.450808:0:42258:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788877040.450814:0:42258:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788877040.450823:0:42258:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:3.0:1788877040.450830:0:42258:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:3.0:1788877040.450961:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.450975:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00800000:00000004:3.0:1788877040.450983:0:42258:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 2022, max_age 2031 00000100:00100000:3.0:1788877040.451025:0:42258:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318400:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788877040.451110:0:42258:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877040.457260:0:42258:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318400:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788877040.457287:0:42258:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023705/1024000 00000080:00200080:3.0:1788877040.457352:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.457367:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00800000:00000004:3.0:1788877040.457373:0:42258:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 2022, max_age 2031 00000100:00100000:3.0:1788877040.457430:0:42258:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318528:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788877040.457498:0:42258:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877040.460468:0:42258:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318528:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788877040.460494:0:42258:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:3.0:1788877040.460549:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460564:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460571:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460574:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460578:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460581:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460584:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460587:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460590:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460593:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460597:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460600:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460603:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460606:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460610:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460613:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460616:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460619:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460622:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460625:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460628:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460631:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460634:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460638:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460641:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460644:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460647:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460650:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460653:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460655:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460659:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460661:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460664:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460667:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460670:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460672:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460675:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460678:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460681:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460684:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460687:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460690:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460693:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460696:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460699:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460701:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460705:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460708:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460713:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460717:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460721:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460724:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460727:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460730:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460734:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460737:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460741:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460744:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460748:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460751:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460755:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460759:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460763:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460767:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460771:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460775:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460779:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460782:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460785:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460789:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460793:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460797:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460800:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460805:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460808:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460812:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460815:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460819:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460823:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460826:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460829:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460833:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460837:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460840:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460844:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460848:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460851:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460855:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460858:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460862:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460865:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460868:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460872:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460875:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460879:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460882:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460886:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460890:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460894:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460898:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460902:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460905:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460908:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460912:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460915:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460919:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460922:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460926:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460930:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460933:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460937:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460941:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460944:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460948:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460952:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460955:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460959:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460962:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460966:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460969:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460972:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460975:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460979:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460982:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460985:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00800000:00000080:3.0:1788877040.460989:0:42258:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00000080:00200080:3.0:1788877040.460993:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:3.0:1788877040.460999:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa06188389800 uarg=0000000000000000 00020000:00000004:3.0:1788877040.461007:0:42258:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 2022, max_age 2031 00000100:00100000:3.0:1788877040.461037:0:42258:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318656:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788877040.461093:0:42258:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877040.466275:0:42258:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318656:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788877040.466312:0:42258:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788877040.466374:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.467057:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00020000:00000004:2.0:1788877040.467067:0:42258:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 2022, max_age 2031 00000100:00100000:2.0:1788877040.467107:0:42258:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318784:192.168.203.111@tcp:13:lfs.0 00000100:00100000:2.0:1788877040.467179:0:42258:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061860f3180 going to sleep for 11 seconds 00000100:00100000:2.0:1788877040.480806:0:42258:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42258:1875771511318784:192.168.203.111@tcp:13:lfs.0 00020000:00000004:2.0:1788877040.480833:0:42258:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:2.0:1788877040.480902:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480919:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480926:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480931:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480936:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480939:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480943:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480947:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480951:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480954:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480958:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480962:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480966:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480970:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480974:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480978:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480981:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480985:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480988:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480991:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.480995:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.480998:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481002:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481005:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481009:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481012:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481016:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481020:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481023:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481027:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481031:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481035:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481039:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481043:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481046:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481050:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481054:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481057:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481061:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481064:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481067:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481070:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481075:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481078:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481081:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481084:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481087:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481090:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481094:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481097:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481100:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481103:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481106:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481109:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481113:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481116:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481121:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481124:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481128:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481131:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481135:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481138:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481142:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481145:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481148:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481151:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481154:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481157:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481161:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481163:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481166:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481169:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481173:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481175:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481179:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481181:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481185:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481187:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481191:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481194:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481198:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481200:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481204:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481207:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481210:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481213:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481216:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481219:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481222:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481225:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481228:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481231:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481235:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481237:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481240:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481243:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481246:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481248:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481252:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481254:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481258:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481260:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481263:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481266:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481270:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481273:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481290:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481295:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481298:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481301:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481304:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481307:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481310:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481313:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481316:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481319:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481322:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481325:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481328:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481331:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481334:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481337:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481340:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481343:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200080:2.0:1788877040.481347:0:42258:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffd07e1bc70 00020000:00000080:2.0:1788877040.481350:0:42258:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa061a1e06c00 uarg=0000000000000000 00000080:00200008:2.0:1788877040.481363:0:42258:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:1.0:1788877040.549534:0:42276:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788877040.549561:0:42276:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788877040.549579:0:42276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1788877040.549599:0:42276:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:1.0:1788877040.549608:0:42276:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788877040.549615:0:42276:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788877040.549618:0:42276:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:1.0:1788877040.549625:0:42276:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac1463b58 00010000:00010000:1.0:1788877040.549629:0:42276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:1.0:1788877040.549638:0:42276:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:1.0:1788877040.549654:0:42276:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788877040.549668:0:42276:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:1.0:1788877040.549672:0:42276:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f10.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:1.0:1788877040.549684:0:42276:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f10.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:1.0:1788877040.549726:0:42276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.549733:0:42276:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.549742:0:42276:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788877040.549766:0:42276:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:42276:1875771511318912:192.168.203.111@tcp:101:checkstat.0 00000100:00100000:1.0:1788877040.549845:0:42276:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061a1ed6900 going to sleep for 11 seconds 00000100:00100000:1.0:1788877040.560173:0:42276:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:42276:1875771511318912:192.168.203.111@tcp:101:checkstat.0 00010000:00010000:1.0:1788877040.560197:0:42276:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560211:0:42276:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560218:0:42276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560225:0:42276:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560233:0:42276:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560240:0:42276:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618902ec00 already unused, calling callback (ffffffffc19a4630) 00010000:00010000:1.0:1788877040.560244:0:42276:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560251:0:42276:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560259:0:42276:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560266:0:42276:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:1.0:1788877040.560273:0:42276:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.560285:0:42276:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902ec00/0x7d14dcdc37048a69 lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 42276 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788877040.560302:0:42276:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffffa0618869a680 x1875771511318912/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788877056 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'checkstat.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788877040.560317:0:42276:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:1.0:1788877040.560346:0:42276:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1788877040.560357:0:42276:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00010000:1.0:1788877040.560366:0:42276:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac1463930 00010000:00010000:1.0:1788877040.560369:0:42276:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:1.0:1788877040.560376:0:42276:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000100:00100000:1.0:1788877040.578524:0:42278:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job checkstat:f0204769-56bd-4f5c-ab97-74c450c9e072:42278:1875771511319040:192.168.203.111@tcp:101:checkstat.0 00010000:00010000:1.0:1788877040.578554:0:42278:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578576:0:42278:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578585:0:42278:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578597:0:42278:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578608:0:42278:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578620:0:42278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578639:0:42278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578646:0:42278:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788877040.578657:0:42278:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06198afc000 x1875771511319040/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788877056 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'checkstat.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788877040.578671:0:42278:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578682:0:42278:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/275 (0 0) 00000080:00200000:1.0:1788877040.578689:0:42278:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:1.0:1788877040.578708:0:42278:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x113:0x0], updating i_size 0 00000080:00200000:1.0:1788877040.578712:0:42278:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x113:0x0](ffffa061981be3c8): rc = 0 00000080:00200000:1.0:1788877040.578727:0:42278:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x113:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:1.0:1788877040.578732:0:42278:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x113:0x0](ffffa061981be3c8) for lock 0x7d14dcdc37048a70 00000080:00010000:1.0:1788877040.578742:0:42278:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x113:0x0](ffffa061981be3c8) 00000080:00010000:1.0:1788877040.578746:0:42278:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a70 from it ffffb95ac163bb20 00010000:00010000:1.0:1788877040.578749:0:42278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877040.578759:0:42278:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:1.0:1788877040.578795:0:42278:0:(file.c:6296:ll_getattr_dentry()) START file f10.replay-single-2:[0x200001b71:0x113:0x0](ffffa061981be3c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1788877040.578800:0:42278:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x113:0x0](ffffa061981be3c8),name=f10.replay-single-2 00000002:00010000:1.0:1788877040.578821:0:42278:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x113:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788877040.578829:0:42278:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788877040.578838:0:42278:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788877040.578844:0:42278:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788877040.578849:0:42278:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x113:0x0](ffffa061981be3c8) 00000080:00010000:1.0:1788877040.578851:0:42278:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x113:0x0](ffffa061981be3c8) for lock 0x7d14dcdc37048a70 00000080:00010000:1.0:1788877040.578855:0:42278:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a70 from it ffffb95ac163bcb8 00010000:00010000:1.0:1788877040.578858:0:42278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788877040.578865:0:42278:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886ce200/0x7d14dcdc37048a70 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x113:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b093d expref: -99 pid: 42278 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788877040.583171:0:42280:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788877040.583199:0:42280:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788877040.583217:0:42280:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1788877040.583232:0:42280:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:1.0:1788877040.583240:0:42280:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788877040.583246:0:42280:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788877040.583250:0:42280:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a62 00000080:00010000:1.0:1788877040.583256:0:42280:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a62 from it ffffb95ac0d87b58 00010000:00010000:1.0:1788877040.583260:0:42280:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:1.0:1788877040.583269:0:42280:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cea00/0x7d14dcdc37048a62 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b090c expref: -99 pid: 41790 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:1.0:1788877040.585839:0:42280:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788877040.585856:0:42280:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f10.replay-single-2, flags=8 00000100:00100000:3.0:1788877040.596795:0:42280:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:42280:1875771511319296:192.168.203.111@tcp:36:rm.0 00000080:00200000:3.0:1788877040.596851:0:42280:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x113:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788877040.596867:0:42280:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x113:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788877040.596875:0:42280:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x113:0x0](ffffa061981be3c8) 00010000:00010000:1.0:1788877041.407190:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877041.803475:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511319424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.803587:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319680:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.803615:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319936:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.803642:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:1.0:1788877041.803649:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.803670:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319808:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.806928:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319808:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.806944:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319936:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.810278:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.810294:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2395:1875771511319424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877041.810304:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511319680:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877042.438162:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877045.502636:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877046.530345:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788877046.910618:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511320064:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877046.910695:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320320:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877046.910717:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320576:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877046.910741:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1788877046.910747:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320192:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877046.910764:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320448:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877046.914081:0:2397:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 94489280513 gen 1 00000100:00100000:3.0:1788877046.914110:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320192:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877046.914130:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320320:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877046.923073:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320448:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877046.923090:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511320064:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877046.923102:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511320576:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788877047.552642:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1788877047.760024:0:42850:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: == replay-single test 11: create open write rename |X| create-old-name read ========================================================== 10:17:26 (1788877046) 00000080:00200000:3.0:1788877047.931639:0:42856:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877047.931668:0:42856:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877047.931724:0:42856:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42856 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877047.931735:0:42856:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42856 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877047.931745:0:42856:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42856 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877047.931771:0:42856:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:42856:1875771511320704:192.168.203.111@tcp:101:mcreate.0 00000100:00100000:3.0:1788877047.932755:0:42856:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061891689c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788877047.969564:0:42856:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:42856:1875771511320704:192.168.203.111@tcp:101:mcreate.0 00010000:00010000:1.0:1788877047.969592:0:42856:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969604:0:42856:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969617:0:42856:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969628:0:42856:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969640:0:42856:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969650:0:42856:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969659:0:42856:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:1.0:1788877047.969670:0:42856:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188872300 x1875771511320704/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877063 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:0 00000002:00010000:1.0:1788877047.969683:0:42856:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969697:0:42856:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:1.0:1788877047.969703:0:42856:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:1.0:1788877047.969728:0:42856:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:1.0:1788877047.969734:0:42856:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a77 00000080:00010000:1.0:1788877047.969739:0:42856:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788877047.969742:0:42856:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a77 from it ffffb95ac1463bd0 00010000:00010000:1.0:1788877047.969746:0:42856:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877047.969755:0:42856:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788877047.969780:0:42856:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:1.0:1788877047.969797:0:42856:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:1.0:1788877047.969802:0:42856:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788877047.969816:0:42856:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788877047.969825:0:42856:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788877047.969840:0:42856:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1788877047.969848:0:42856:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788877047.969853:0:42856:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788877047.969855:0:42856:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a77 00000080:00010000:1.0:1788877047.969859:0:42856:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a77 from it ffffb95ac1463cd0 00010000:00010000:1.0:1788877047.969862:0:42856:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1788877047.969870:0:42856:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788877047.969879:0:42856:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788877047.969885:0:42856:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788877047.969890:0:42856:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788877047.969896:0:42856:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788877047.969904:0:42856:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788877047.969911:0:42856:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1788877047.969914:0:42856:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:1.0:1788877047.969917:0:42856:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a77 00000080:00010000:1.0:1788877047.969920:0:42856:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a77 from it ffffb95ac1463db0 00010000:00010000:1.0:1788877047.969924:0:42856:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788877047.969932:0:42856:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:1.0:1788877047.969939:0:42856:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:1.0:1788877047.969944:0:42856:0:(namei.c:2062:ll_create_nd()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:1.0:1788877047.969948:0:42856:0:(namei.c:2017:ll_mknod()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:1.0:1788877047.969971:0:42856:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788877047.970010:0:42856:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:1.0:1788877047.970017:0:42856:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1788877047.970020:0:42856:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b200/0x7d14dcdc37048a77 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0967 expref: -99 pid: 42856 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:1.0:1788877047.970042:0:42856:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:1.0:1788877047.970049:0:42856:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa0618869a680 x1875771511320960/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877047.970070:0:42856:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:42856:1875771511320960:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:1.0:1788877047.970144:0:42856:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10780 going to sleep for 11 seconds 00000100:00100000:1.0:1788877047.980108:0:42856:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:42856:1875771511320960:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:1.0:1788877047.980166:0:42856:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x115:0x0](ffffa061981c6c08) 00000080:00200000:1.0:1788877047.980205:0:42856:0:(namei.c:2070:ll_create_nd()) VFS Op:name=f11.replay-single, unhashed 1 00000080:00200000:2.0:1788877047.981352:0:42718:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877047.981374:0:42718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877047.981467:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.981477:0:42718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.981487:0:42718:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788877047.981509:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321088:192.168.203.111@tcp:101:bash.0 00000100:00100000:2.0:1788877047.981578:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cfff00 going to sleep for 11 seconds 00000100:00100000:2.0:1788877047.989375:0:42718:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321088:192.168.203.111@tcp:101:bash.0 00010000:00010000:2.0:1788877047.989618:0:42718:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.989631:0:42718:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.989642:0:42718:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.989653:0:42718:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.989667:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.989674:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.989681:0:42718:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788877047.989693:0:42718:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888ac000 x1875771511321088/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877063 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788877047.989706:0:42718:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877047.989717:0:42718:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788877047.989723:0:42718:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788877047.990648:0:42718:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788877047.990659:0:42718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a7e 00000080:00010000:3.0:1788877047.990666:0:42718:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877047.990669:0:42718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a7e from it ffffb95ac1823bf8 00010000:00010000:3.0:1788877047.990673:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877047.990685:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877047.990711:0:42718:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877047.990727:0:42718:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877047.990742:0:42718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877047.990753:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877047.990764:0:42718:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877047.990772:0:42718:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877047.990776:0:42718:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877047.990779:0:42718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048a7e 00000080:00010000:3.0:1788877047.990783:0:42718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a7e from it ffffb95ac1823b38 00010000:00010000:3.0:1788877047.990786:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877047.990794:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788877047.990803:0:42718:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788877047.990808:0:42718:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa06198ade480, open_flags 8241, mode 81b6 opened 0 00000080:00200000:3.0:1788877047.990817:0:42718:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open|creat 00000002:00010000:3.0:1788877047.990838:0:42718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f11.replay-single,[0x200001b71:0x116:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:3.0:1788877047.990850:0:42718:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877047.990891:0:42718:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877047.990904:0:42718:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788877047.990907:0:42718:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030800/0x7d14dcdc37048a7e lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0983 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877047.990941:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048a85 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877047.990947:0:42718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048a85 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1788877047.990957:0:42718:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1788877047.990966:0:42718:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188873b80 x1875771511321344/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1788877047.990975:0:42718:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887aa00/0x7d14dcdc37048a85 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877047.990992:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321344:192.168.203.111@tcp:101:bash.0 00000100:00100000:3.0:1788877047.991163:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa061891686c0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877048.011618:0:42718:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321472:192.168.203.111@tcp:101:bash.0 00010000:00010000:2.0:1788877048.011645:0:42718:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877048.011661:0:42718:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877048.011674:0:42718:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788877048.011694:0:42718:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x80 disposition=0x0, status=0 req@ffffa06198afc380 x1875771511321472/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 568/1112 e 0 to 0 dl 1788877064 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788877048.011708:0:42718:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877048.011722:0:42718:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/277 (0 0) 00000080:00200000:2.0:1788877048.011728:0:42718:0:(namei.c:952:ll_intent_lock()) intent lock 128 on i1 [0x200001b71:0x115:0x0] suppgids 0 -1: rc 0 00000080:00010000:2.0:1788877048.011752:0:42718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffffa061981c6c08) for lock 0x7d14dcdc37048a8c 00000080:00010000:2.0:1788877048.011760:0:42718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048a8c from it ffffb95ac1823a38 00010000:00010000:2.0:1788877048.011765:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877048.011776:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788877048.011810:0:42718:0:(llite_lib.c:2427:ll_setattr_raw()) START file f11.replay-single:[0x200001b71:0x115:0x0](ffffa061981c6c08) current size 0, valid attrs a068, mode 0, uid 2, gid 0, new size 0, atime 1788877048.000000000, mtime 1788877048.000000000, ctime 1788877048.000000000 00000020:00100000:2.0:1788877048.011857:0:42718:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788877048.011864:0:42718:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ad880 x1875771511321728/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 488/512 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877048.011883:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321728:192.168.203.111@tcp:36:bash.0 00000100:00100000:2.0:1788877048.011949:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cfff00 going to sleep for 11 seconds 00000100:00100000:2.0:1788877048.017612:0:42718:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321728:192.168.203.111@tcp:36:bash.0 00000080:00200000:2.0:1788877048.017660:0:42718:0:(file.c:1848:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000080:00200000:2.0:1788877048.017667:0:42718:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00010000:00010000:2.0:1788877048.017686:0:42718:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 31 res 8589941617/277 (0 0) 00000080:00010000:2.0:1788877048.017697:0:42718:0:(file.c:7065:ll_layout_intent()) ### lustre: requeue layout lock for file [0x200001b71:0x115:0x0](ffffa061981c6c08) 00000002:00010000:2.0:1788877048.017707:0:42718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x115:0x0]) in obj [0x200001b71:0x115:0x0], intent: layout flags 0 00010000:00010000:2.0:1788877048.017750:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048a93 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877048.017759:0:42718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048a93 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877048.017768:0:42718:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048a93 lrc: 3/1,0 mode: --/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788877048.017786:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321856:192.168.203.111@tcp:101:bash.0 00000100:00100000:2.0:1788877048.017843:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cfff00 going to sleep for 11 seconds 00000100:00100000:1.0:1788877048.032583:0:42718:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff100 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511321984:192.168.203.111@tcp:101:bash.0 00010000:00010000:1.0:1788877048.032606:0:42718:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877048.032621:0:42718:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x80000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877048.032634:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10000080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:1.0:1788877048.032646:0:42718:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 0, setting rss=0; kms=0, end=18446744073709551615 ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,2 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x10020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877048.032658:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877048.032665:0:42718:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788877048.032744:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788877048.032757:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511322112:192.168.203.111@tcp:10:bash.0 00000100:00100000:1.0:1788877048.041728:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511322112:192.168.203.111@tcp:10:bash.0 00000080:00200000:3.0:1788877048.041867:0:42718:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 0 i_blocks 0 00000020:00010000:3.0:1788877048.041875:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c548a8 (ffffa06189032878/0) at cl_lock_release():245 00000020:00010000:3.0:1788877048.041879:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c548a8 (ffffa06189032878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788877048.041883:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa061864d5ab0 (ffffa061890326c8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788877048.041895:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.041910:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:3.0:1788877048.041933:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c548a8 (ffffa06189032878/0) at cl_lock_fini():72 00000020:00010000:3.0:1788877048.041936:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa061864d5ab0 (ffffa06189032878/1) at cl_lock_fini():72 00000080:00200000:3.0:1788877048.041952:0:42718:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1788877048.041964:0:42718:0:(llite_lib.c:2686:ll_setattr_raw()) COMPLETED file f11.replay-single:[0x200001b71:0x115:0x0](ffffa061981c6c08) current size 0, valid attrs 1000a168, mode 0, uid 2, gid 0, new size 0, atime 1788877048.000000000, mtime 1788877048.000000000, ctime 1788877048.000000000, rc 0 00000080:00200008:3.0:1788877048.042035:0:42718:0:(file.c:2650:do_file_write_iter()) START file f11.replay-single:[0x200001b71:0x115:0x0], ppos: 0, count: 4 00000080:00200000:3.0:1788877048.042047:0:42718:0:(rw26.c:676:ll_write_begin()) Writing 0 of 0 to 4 bytes 00000080:00200000:3.0:1788877048.042053:0:42718:0:(file.c:2626:ll_do_tiny_write()) result: 0, original count 4 00000080:00200000:3.0:1788877048.042057:0:42718:0:(file.c:2090:ll_file_io_generic()) f11.replay-single: write ppos: 0, bytes: 4 00000080:00200000:3.0:1788877048.042061:0:42718:0:(file.c:1848:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:3.0:1788877048.042067:0:42718:0:(cl_io.c:214:cl_io_rw_init()) header@000000003825996c[0x110, 1, [0x200001b71:0x115:0x0] hash] 00000020:00200000:3.0:1788877048.042068:0:42718:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4) 0 0 00000080:00200000:3.0:1788877048.042072:0:42718:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788877048.042082:0:42718:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:3.0:1788877048.042088:0:42718:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4) 4, 4 00020000:00200000:3.0:1788877048.042091:0:42718:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788877048.042104:0:42718:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:3.0:1788877048.042126:0:42718:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788877048.042131:0:42718:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9dc30, osc lock ffffa061885be4c0, flags 0x0 00010000:00010000:3.0:1788877048.042149:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.042162:0:42718:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.042174:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/0,2 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.042187:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/0,2 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.042195:0:42718:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877048.042206:0:42718:0:(vvp_io.c:1292:vvp_io_write_start()) f11.replay-single: write [0, 4) append 0 append lockpos 0 00000080:00200000:3.0:1788877048.042214:0:42718:0:(rw26.c:676:ll_write_begin()) Writing 0 of 0 to 4 bytes 00000080:00200000:3.0:1788877048.042268:0:42718:0:(rw26.c:871:ll_write_end()) pos 0, len 4, copied 4 00000080:00200000:3.0:1788877048.042276:0:42718:0:(rw26.c:929:ll_write_end()) page@000000001d565204[3 00000000946dc24d 2 1 00000000dfdcdfdd 0 0 0] 00000080:00200000:3.0:1788877048.042279:0:42718:0:(rw26.c:929:ll_write_end()) vmpage @00000000b645c038 17ffffc0001001 3:0 ffffa061903c6e00 0 lru 00000080:00200000:3.0:1788877048.042287:0:42718:0:(rw26.c:929:ll_write_end()) osc-page@000000005d3506a6 0: 1< 0 - - > 2< 0 0 0 0x0 0x0 | 000000004faa384e 00000000e192417e > 3< 0 0 > 4< 0 0 8 491642880 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:3.0:1788877048.042289:0:42718:0:(rw26.c:929:ll_write_end()) end page@000000001d565204 00000080:00200000:3.0:1788877048.042291:0:42718:0:(rw26.c:929:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1788877048.042295:0:42718:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 1, from 0, to 4 prio 0 00000008:00010000:3.0:1788877048.042312:0:42718:0:(osc_cache.c:858:osc_extent_find()) ### extent: 00000000fd1a227c ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877048.042328:0:42718:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffffa061981c6db8, count 1, dirtied 1 00000080:00200000:3.0:1788877048.042332:0:42718:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 1 pages 4 bytes, tot: 4 00000080:00200000:3.0:1788877048.042334:0:42718:0:(vvp_io.c:1358:vvp_io_write_start()) f11.replay-single: write bytes 0, result: 4 00000020:00010000:3.0:1788877048.042345:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c548a8 (ffffa06189032878/0) at cl_lock_release():245 00000020:00010000:3.0:1788877048.042348:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c548a8 (ffffa06189032878/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788877048.042350:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9dc30 (ffffa061890326c8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788877048.042355:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.042367:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00010000:3.0:1788877048.042375:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c548a8 (ffffa06189032878/0) at cl_lock_fini():72 00000020:00010000:3.0:1788877048.042378:0:42718:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9dc30 (ffffa06189032878/1) at cl_lock_fini():72 00000080:00200000:3.0:1788877048.042416:0:42718:0:(vvp_io.c:533:vvp_io_advance()) advancing 4 bytes 00000080:00200000:3.0:1788877048.042421:0:42718:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 0] 00000080:00200000:3.0:1788877048.042426:0:42718:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788877048.042430:0:42718:0:(file.c:2238:ll_file_io_generic()) f11.replay-single: 2 io complete with rc: 0, result: 4, restart: 0 00000080:00200000:3.0:1788877048.042435:0:42718:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 0 00000080:00200000:3.0:1788877048.042438:0:42718:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4 00000080:00200008:3.0:1788877048.042453:0:42718:0:(file.c:479:ll_file_release()) START file f11.replay-single:[0x200001b71:0x115:0x0](ffffa061981c6c08), flags 100001 00000002:00100000:3.0:1788877048.042505:0:42718:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa06188873b80 x1875771511321344/t94489280515(94489280515) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788877063 ref 1 fl Complete:RPQU/604/ffffffff rc 301/-1 job:'bash.0' uid:0 gid:0 projid:0 00000020:00100000:3.0:1788877048.042516:0:42718:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:3.0:1788877048.042521:0:42718:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188870e00 x1875771511322368/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:3.0:1788877048.042541:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511322368:192.168.203.111@tcp:35:bash.0 00000100:00100000:3.0:1788877048.042607:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877048.050940:0:42718:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511322368:192.168.203.111@tcp:35:bash.0 00000002:00100000:3.0:1788877048.050977:0:42718:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa06188873b80 x1875771511321344/t94489280515(94489280515) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788877063 ref 1 fl Complete:RQU/604/ffffffff rc 301/-1 job:'bash.0' uid:0 gid:0 projid:0 00000080:00200000:1.0:1788877048.063877:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:1.0:1788877048.063905:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1788877048.063967:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877048.063979:0:42863:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1788877048.063989:0:42863:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1788877048.064012:0:42863:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511322496:192.168.203.111@tcp:101:mv.0 00000100:00100000:1.0:1788877048.064093:0:42863:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e10780 going to sleep for 11 seconds 00000100:00100000:3.0:1788877048.075965:0:42863:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511322496:192.168.203.111@tcp:101:mv.0 00010000:00010000:3.0:1788877048.075995:0:42863:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076006:0:42863:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076018:0:42863:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076030:0:42863:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076043:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076052:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076059:0:42863:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877048.076071:0:42863:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa0618869bb80 x1875771511322496/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877064 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788877048.076084:0:42863:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076096:0:42863:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788877048.076102:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788877048.076141:0:42863:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788877048.076146:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:3.0:1788877048.076152:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877048.076155:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fbd0 00010000:00010000:3.0:1788877048.076158:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076167:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877048.076190:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877048.076216:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877048.076231:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877048.076239:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.076249:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877048.076255:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877048.076259:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877048.076262:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:3.0:1788877048.076265:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fbd0 00010000:00010000:3.0:1788877048.076268:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.076275:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788877048.076287:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877048.076302:0:42863:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=0 00000080:00200000:3.0:1788877048.076306:0:42863:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788877048.076318:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f11.replay-single,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788877048.076354:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076361:0:42863:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.076371:0:42863:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877048.076438:0:42863:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511322624:192.168.203.111@tcp:101:mv.0 00000100:00100000:3.0:1788877048.076502:0:42863:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877048.086665:0:42863:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511322624:192.168.203.111@tcp:101:mv.0 00010000:00010000:3.0:1788877048.086691:0:42863:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086707:0:42863:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x2/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086715:0:42863:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086726:0:42863:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086737:0:42863:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086749:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086757:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086764:0:42863:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877048.086775:0:42863:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188870000 x1875771511322624/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/640 e 0 to 0 dl 1788877064 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'mv.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788877048.086785:0:42863:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: getattr, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1788877048.086796:0:42863:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086805:0:42863:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/277 (0 0) 00000080:00200000:3.0:1788877048.086811:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788877048.086853:0:42863:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x115:0x0](ffffa061981c6c08): rc = 0 00000080:00200000:3.0:1788877048.086865:0:42863:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x115:0x0]: layout version change: 0 -> 0 00000080:00010000:3.0:1788877048.086869:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffffa061981c6c08) for lock 0x7d14dcdc37048aa8 00000080:00010000:3.0:1788877048.086878:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffffa061981c6c08) 00000080:00010000:3.0:1788877048.086882:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa8 from it ffffb95ac1d6fdb8 00010000:00010000:3.0:1788877048.086885:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086893:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877048.086933:0:42863:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=2048 00000080:00200000:3.0:1788877048.086937:0:42863:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:3.0:1788877048.086957:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f11.replay-single-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:3.0:1788877048.086992:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.086999:0:42863:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.087007:0:42863:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877048.087025:0:42863:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511322752:192.168.203.111@tcp:101:mv.0 00000100:00100000:3.0:1788877048.087094:0:42863:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877048.096789:0:42863:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511322752:192.168.203.111@tcp:101:mv.0 00010000:00010000:3.0:1788877048.096817:0:42863:0:(ldlm_request.c:688:ldlm_cli_enqueue_fini()) ### client-side enqueue END (ABORTED) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096831:0:42863:0:(ldlm_request.c:591:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096839:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096849:0:42863:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x10206400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096859:0:42863:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x206400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096866:0:42863:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa0618887a000 already unused, calling callback (ffffffffc19a4630) 00010000:00010000:3.0:1788877048.096871:0:42863:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096879:0:42863:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 4/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x206400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096887:0:42863:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 5/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x20e400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096894:0:42863:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:3.0:1788877048.096901:0:42863:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 3/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877048.096908:0:42863:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048aaf lrc: 0/0,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x2 rrc: 3 type: IBT flags: 0x4a0f400000000 pid: 42863 initiator: MDT0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877048.096926:0:42863:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0x7, status=0 req@ffffa06188870000 x1875771511322752/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/584 e 0 to 0 dl 1788877064 ref 1 fl Complete:RQU/600/0 rc 301/301 job:'mv.0' uid:0 gid:0 projid:0 00000080:00200000:3.0:1788877048.096940:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00010000:00010000:3.0:1788877048.096969:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877048.096982:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00010000:3.0:1788877048.096992:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fbc8 00010000:00010000:3.0:1788877048.096997:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877048.097006:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788877048.097037:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877048.097051:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877048.097059:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877048.097067:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788877048.097074:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877048.097079:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877048.097082:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:3.0:1788877048.097086:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fc98 00010000:00010000:3.0:1788877048.097090:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877048.097098:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788877048.097106:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788877048.097112:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877048.097118:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877048.097124:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788877048.097133:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788877048.097140:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877048.097144:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877048.097146:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:3.0:1788877048.097150:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fcc8 00010000:00010000:3.0:1788877048.097153:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788877048.097162:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788877048.097170:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788877048.097210:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877048.097216:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877048.097222:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788877048.097229:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788877048.097236:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877048.097253:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877048.097255:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:3.0:1788877048.097260:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fb58 00010000:00010000:3.0:1788877048.097263:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788877048.097270:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788877048.097279:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877048.097285:0:42863:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single-2, flags=8 00000080:00200000:3.0:1788877048.097300:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877048.097306:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877048.097312:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788877048.097320:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788877048.097327:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877048.097331:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877048.097333:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:3.0:1788877048.097336:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fbd0 00010000:00010000:3.0:1788877048.097339:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788877048.097347:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:3.0:1788877048.097355:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877048.097362:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877048.097367:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877048.097373:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1788877048.097647:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00000080:00200000:2.0:1788877048.097656:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877048.097662:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877048.097665:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:2.0:1788877048.097669:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fbd0 00010000:00010000:2.0:1788877048.097673:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 0 00010000:00010000:2.0:1788877048.097680:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1788877048.097690:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788877048.097699:0:42863:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single, flags=0 00000080:00200000:2.0:1788877048.097702:0:42863:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single-2, flags=2048 00000080:00200000:2.0:1788877048.097705:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877048.097712:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877048.097719:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1788877048.097727:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00000080:00200000:2.0:1788877048.097733:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877048.097738:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877048.097740:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:2.0:1788877048.097744:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fc98 00010000:00010000:2.0:1788877048.097747:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 0 00010000:00010000:2.0:1788877048.097755:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1788877048.097764:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788877048.097768:0:42863:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877048.097774:0:42863:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877048.097780:0:42863:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1788877048.097788:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00000080:00200000:2.0:1788877048.097800:0:42863:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877048.097804:0:42863:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877048.097807:0:42863:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048aa1 00000080:00010000:2.0:1788877048.097810:0:42863:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aa1 from it ffffb95ac1d6fcc8 00010000:00010000:2.0:1788877048.097814:0:42863:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 0 00010000:00010000:2.0:1788877048.097822:0:42863:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000080:00200000:2.0:1788877048.097830:0:42863:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788877048.097836:0:42863:0:(namei.c:2425:ll_rename()) VFS Op:oldname=f11.replay-single, src_dir=[0x200000007:0x1:0x0](ffffa061981363c8), newname=f11.replay-single-2, tgt_dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:2.0:1788877048.097859:0:42863:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1788877048.097881:0:42863:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877048.097892:0:42863:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030c00/0x7d14dcdc37048a93 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x115:0x0].0x0 bits 0x8/0x0 rrc: 5 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b09c2 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788877048.097927:0:42863:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877048.097935:0:42863:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00010000:00010000:2.0:1788877048.097947:0:42863:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1788877048.097950:0:42863:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048aa8 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313b09f3 expref: -99 pid: 42863 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877048.097961:0:42863:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618902e600/0x7d14dcdc37048aa1 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b09d7 expref: -99 pid: 42863 timeout: 0 lvb_type: 0 lru_score: 10 lru_type: 0 00000020:00100000:2.0:1788877048.097982:0:42863:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788877048.097990:0:42863:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888aea00 x1875771511323008/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:13/10 lens 536/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877048.098011:0:42863:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511323008:192.168.203.111@tcp:36:mv.0 00000100:00100000:2.0:1788877048.098083:0:42863:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cff6c0 going to sleep for 11 seconds 00000100:00100000:1.0:1788877048.104018:0:2387:0:(events.c:344:request_in_callback()) peer: 12345-192.168.203.111@tcp (source: 12345-192.168.203.111@tcp) 00010000:00010000:3.0:1788877048.104535:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1788877048.104542:0:15191:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000000 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.104551:0:15191:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffffa061886cec00 already unused, calling callback (ffffffffc19a4630) 00010000:00010000:3.0:1788877048.104556:0:15191:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.104565:0:15191:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x400400000000 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.104574:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 3/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x408400000000 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.104625:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 2/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.104633:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788877048.104657:0:15191:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.104665:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa061886cec00/0x7d14dcdc37048a8c lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x20/0x20 rrc: 3 type: IBT gid 0 flags: 0x4c09400000020 nid: local remote: 0xf5bf0f7e313b09b4 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877048.104685:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877048.104688:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877048.104690:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788877048.104698:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788877048.104712:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511323136:192.168.203.111@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1788877048.115466:0:42863:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job mv:f0204769-56bd-4f5c-ab97-74c450c9e072:42863:1875771511323008:192.168.203.111@tcp:36:mv.0 00000100:00100000:3.0:1788877048.116323:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511323136:192.168.203.111@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1788877048.573674:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:3.0:1788877049.084576:0:42872:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788877049.084590:0:42872:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877049.084612:0:42872:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877049.084672:0:42872:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42872 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.084683:0:42872:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42872 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.084694:0:42872:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42872 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877049.084718:0:42872:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323264:192.168.203.111@tcp:101:lfs.0 00000100:00100000:3.0:1788877049.084804:0:42872:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877049.096230:0:42872:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 94489280519 gen 1 00000100:00100000:3.0:1788877049.096280:0:42872:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323264:192.168.203.111@tcp:101:lfs.0 00010000:00010000:3.0:1788877049.096300:0:42872:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096312:0:42872:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096325:0:42872:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096337:0:42872:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096349:0:42872:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096357:0:42872:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096365:0:42872:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877049.096376:0:42872:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188870000 x1875771511323264/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877065 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788877049.096431:0:42872:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096445:0:42872:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788877049.096451:0:42872:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788877049.096477:0:42872:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788877049.096484:0:42872:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ab6 00000080:00010000:3.0:1788877049.096489:0:42872:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877049.096493:0:42872:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ab6 from it ffffb95ac163bcb8 00010000:00010000:3.0:1788877049.096497:0:42872:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877049.096507:0:42872:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:3.0:1788877049.097326:0:42872:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), request_mask 0, flags 0, foreign 0 00000080:00200000:3.0:1788877049.097333:0:42872:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877049.097349:0:42872:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877049.097359:0:42872:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877049.097371:0:42872:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877049.097377:0:42872:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877049.097411:0:42872:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877049.097415:0:42872:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ab6 00000080:00010000:3.0:1788877049.097420:0:42872:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ab6 from it ffffb95ac163bcb8 00010000:00010000:3.0:1788877049.097423:0:42872:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877049.097430:0:42872:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788877049.097526:0:42872:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877049.097535:0:42872:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877049.097542:0:42872:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877049.097552:0:42872:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788877049.097559:0:42872:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877049.097563:0:42872:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877049.097566:0:42872:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ab6 00000080:00010000:3.0:1788877049.097570:0:42872:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ab6 from it ffffb95ac163bbb0 00010000:00010000:3.0:1788877049.097573:0:42872:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877049.097581:0:42872:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788877049.097590:0:42872:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 44 00000080:00200008:3.0:1788877049.097598:0:42872:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200080:3.0:1788877049.097681:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.097694:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00800000:00000004:3.0:1788877049.097701:0:42872:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: age 2032, max_age 2039 00000100:00100000:3.0:1788877049.097734:0:42872:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323392:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788877049.097807:0:42872:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877049.104493:0:42872:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323392:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788877049.104536:0:42872:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffffa06189393000: update ffffa061893737d8 cache blocks 317388/353529 objects 1023705/1024000 00000080:00200080:3.0:1788877049.104644:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.104674:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00800000:00000004:3.0:1788877049.104682:0:42872:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: age 2032, max_age 2039 00000100:00100000:3.0:1788877049.104716:0:42872:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323520:192.168.203.111@tcp:41:lfs.0 00000100:00100000:3.0:1788877049.104807:0:42872:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877049.116305:0:42872:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323520:192.168.203.111@tcp:41:lfs.0 00800000:00000004:3.0:1788877049.116333:0:42872:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0001-mdc-ffffa06189393000: update ffffa06189370458 cache blocks 317433/353529 objects 1023742/1024000 00000080:00200080:3.0:1788877049.116451:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116469:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116477:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116482:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116486:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116490:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116495:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116499:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116503:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116507:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116510:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116514:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116519:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116522:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116527:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116531:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116535:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116538:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116542:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116546:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116553:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116556:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116560:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116564:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116568:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116571:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116575:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116578:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116582:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116586:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116590:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116594:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116598:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116602:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116606:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116609:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116613:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116617:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116622:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116625:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116629:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116633:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116637:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116641:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116651:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116656:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116660:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116664:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116668:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116672:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116676:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116679:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116683:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116686:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116689:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116692:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116696:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116699:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116702:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116705:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116709:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116713:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116717:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116721:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116725:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116728:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116731:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116735:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116739:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116742:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116747:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116750:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116753:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116757:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116761:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116764:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116768:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116771:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116776:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116779:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116783:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116787:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116791:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116795:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116799:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116803:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116807:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116810:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116814:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116818:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116822:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116826:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116830:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116833:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116837:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116842:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116846:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116849:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116853:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116857:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116861:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116865:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116870:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116873:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116877:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116881:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116885:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116889:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116893:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116896:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116900:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116904:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116908:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116912:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116916:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116920:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116924:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116927:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116932:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116935:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116940:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116944:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116949:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116952:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116956:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00800000:00000080:3.0:1788877049.116961:0:42872:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.116965:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.116971:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00020000:00000004:3.0:1788877049.116979:0:42872:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: age 2032, max_age 2039 00000100:00100000:3.0:1788877049.117010:0:42872:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323648:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788877049.117069:0:42872:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877049.130279:0:42872:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323648:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788877049.130310:0:42872:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffffa06189393000: update ffffa0618858b7d8 cache blocks 901360/958279 objects 261737/262144 00000080:00200080:3.0:1788877049.130421:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.130438:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00020000:00000004:3.0:1788877049.130445:0:42872:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: age 2032, max_age 2039 00000100:00100000:3.0:1788877049.130479:0:42872:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323776:192.168.203.111@tcp:13:lfs.0 00000100:00100000:3.0:1788877049.130536:0:42872:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877049.132440:0:42872:0:(client.c:3126:ptlrpc_free_committed()) lustre-OST0001-osc-ffffa06189393000: committing for last_committed 4294967571 gen 5 00000100:00100000:3.0:1788877049.132462:0:42872:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job lfs:f0204769-56bd-4f5c-ab97-74c450c9e072:42872:1875771511323776:192.168.203.111@tcp:13:lfs.0 00020000:00000004:3.0:1788877049.132475:0:42872:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffffa06189393000: update ffffa06188588458 cache blocks 901364/958279 objects 261737/262144 00000080:00200080:3.0:1788877049.132527:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132539:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132546:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132550:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132553:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132557:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132560:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132563:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132566:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132569:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132572:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132575:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132578:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132581:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132584:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132586:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132589:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132592:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132595:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132599:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132603:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132606:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132610:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132613:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132617:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132620:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132623:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132627:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132631:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132635:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132639:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132642:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132646:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132649:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132653:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132656:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132660:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132664:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132667:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132671:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132675:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132679:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132683:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132686:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132689:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132692:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132696:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132699:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132703:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132706:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132710:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132714:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132718:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132722:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132726:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132730:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132734:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132737:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132741:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132744:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132749:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132752:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132756:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132759:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132763:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132766:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132770:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132773:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132776:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132780:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132784:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132787:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132791:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132794:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132798:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132801:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132804:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132807:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132812:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132815:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132819:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132822:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132826:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132829:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132833:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132837:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132841:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132844:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132849:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132852:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132856:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132859:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132863:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132867:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132871:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132874:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132879:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132882:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132886:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132890:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132894:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132897:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132901:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132905:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132909:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132912:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132916:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132920:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132923:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132927:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132931:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132935:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132939:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132942:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132946:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132949:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132953:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132957:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132961:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132964:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132968:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132971:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132975:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132978:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200080:3.0:1788877049.132982:0:42872:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8) cmd=c00866a4 arg=7ffc1bf56c80 00020000:00000080:3.0:1788877049.132986:0:42872:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffffa06189393000: cmd=c00866a4 len=592 karg=ffffa0618838a800 uarg=0000000000000000 00000080:00200008:3.0:1788877049.132999:0:42872:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffffa061981363c8), flags 100000 00000080:00200000:3.0:1788877049.181557:0:42879:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877049.181589:0:42879:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877049.181610:0:42879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877049.181627:0:42879:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788877049.181634:0:42879:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877049.181641:0:42879:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877049.181644:0:42879:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ab6 00000080:00010000:3.0:1788877049.181651:0:42879:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ab6 from it ffffb95ac0ddbbd0 00010000:00010000:3.0:1788877049.181654:0:42879:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877049.181663:0:42879:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788877049.181682:0:42879:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877049.181698:0:42879:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=1536 00000080:00200000:3.0:1788877049.181701:0:42879:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877049.181707:0:42879:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877049.181712:0:42879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788877049.181719:0:42879:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1788877049.181725:0:42879:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877049.181729:0:42879:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877049.181731:0:42879:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ab6 00000080:00010000:3.0:1788877049.181735:0:42879:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ab6 from it ffffb95ac0ddbcd0 00010000:00010000:3.0:1788877049.181738:0:42879:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788877049.181745:0:42879:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788877049.181752:0:42879:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788877049.181757:0:42879:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877049.181762:0:42879:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877049.181767:0:42879:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788877049.181775:0:42879:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788877049.181781:0:42879:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877049.181783:0:42879:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877049.181785:0:42879:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ab6 00000080:00010000:3.0:1788877049.181788:0:42879:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ab6 from it ffffb95ac0ddbdb0 00010000:00010000:3.0:1788877049.181792:0:42879:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788877049.181798:0:42879:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000080:00200000:3.0:1788877049.181810:0:42879:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788877049.181815:0:42879:0:(namei.c:2062:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=33188, excl=1 00000080:00200000:3.0:1788877049.181819:0:42879:0:(namei.c:2017:ll_mknod()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) mode 100644 dev 0 00010000:00010000:3.0:1788877049.181843:0:42879:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788877049.181890:0:42879:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00010000:00010000:3.0:1788877049.181897:0:42879:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788877049.181900:0:42879:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ab6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0a32 expref: -99 pid: 42872 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 0 00000020:00100000:3.0:1788877049.181924:0:42879:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:3.0:1788877049.181936:0:42879:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188870000 x1875771511324032/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/576 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877049.181960:0:42879:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:42879:1875771511324032:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:3.0:1788877049.182094:0:42879:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877049.187104:0:42879:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job mcreate:f0204769-56bd-4f5c-ab97-74c450c9e072:42879:1875771511324032:192.168.203.111@tcp:36:mcreate.0 00000080:00200000:3.0:1788877049.187176:0:42879:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x117:0x0](ffffa06198283a88) 00000080:00200000:3.0:1788877049.187223:0:42879:0:(namei.c:2070:ll_create_nd()) VFS Op:name=fsa-oleg311-client.virtnet, unhashed 1 00000080:00200000:2.0:1788877049.210618:0:42882:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877049.210649:0:42882:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877049.210713:0:42882:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42882 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.210724:0:42882:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42882 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.210735:0:42882:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42882 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788877049.210760:0:42882:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:42882:1875771511324160:192.168.203.111@tcp:101:rm.0 00000100:00100000:2.0:1788877049.210894:0:42882:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cffcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877049.227975:0:42882:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:42882:1875771511324160:192.168.203.111@tcp:101:rm.0 00010000:00010000:2.0:1788877049.228002:0:42882:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228014:0:42882:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228026:0:42882:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228036:0:42882:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228048:0:42882:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228056:0:42882:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228063:0:42882:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788877049.228073:0:42882:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888af800 x1875771511324160/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877065 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788877049.228086:0:42882:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228097:0:42882:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:2.0:1788877049.228115:0:42882:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:2.0:1788877049.228143:0:42882:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:2.0:1788877049.228148:0:42882:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048abd 00000080:00010000:2.0:1788877049.228154:0:42882:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877049.228156:0:42882:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048abd from it ffffb95ac163bb58 00010000:00010000:2.0:1788877049.228159:0:42882:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228168:0:42882:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788877049.228189:0:42882:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788877049.228205:0:42882:0:(namei.c:1335:ll_lookup_nd()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), flags=8 00000080:00200000:2.0:1788877049.228209:0:42882:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=getattr 00000002:00010000:2.0:1788877049.228229:0:42882:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: fsa-oleg311-client.virtnet,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 01 00010000:00010000:2.0:1788877049.228277:0:42882:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 42882 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228284:0:42882:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42882 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.228293:0:42882:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42882 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788877049.228311:0:42882:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:42882:1875771511324288:192.168.203.111@tcp:101:rm.0 00000100:00100000:2.0:1788877049.228377:0:42882:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cffcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877049.245273:0:42882:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:42882:1875771511324288:192.168.203.111@tcp:101:rm.0 00010000:00010000:2.0:1788877049.245300:0:42882:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245323:0:42882:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245332:0:42882:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245343:0:42882:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245354:0:42882:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245368:0:42882:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245378:0:42882:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245430:0:42882:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788877049.245447:0:42882:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa061888af800 x1875771511324288/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 600/584 e 0 to 0 dl 1788877065 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'rm.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788877049.245464:0:42882:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245479:0:42882:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/279 (0 0) 00000080:00200000:2.0:1788877049.245486:0:42882:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788877049.245508:0:42882:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200001b71:0x117:0x0], updating i_size 0 00000080:00200000:2.0:1788877049.245511:0:42882:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x117:0x0](ffffa06198283a88): rc = 0 00000080:00200000:2.0:1788877049.245528:0:42882:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x117:0x0]: layout version change: 4294967294 -> 4294967295 00000080:00010000:2.0:1788877049.245534:0:42882:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x117:0x0](ffffa06198283a88) for lock 0x7d14dcdc37048ac4 00000080:00010000:2.0:1788877049.245545:0:42882:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x117:0x0](ffffa06198283a88) 00000080:00010000:2.0:1788877049.245549:0:42882:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ac4 from it ffffb95ac163bb20 00010000:00010000:2.0:1788877049.245553:0:42882:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877049.245563:0:42882:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200008:2.0:1788877049.245602:0:42882:0:(file.c:6296:ll_getattr_dentry()) START file fsa-oleg311-client.virtnet:[0x200001b71:0x117:0x0](ffffa06198283a88), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788877049.245609:0:42882:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x117:0x0](ffffa06198283a88),name=fsa-oleg311-client.virtnet 00000002:00010000:2.0:1788877049.245632:0:42882:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x117:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877049.245641:0:42882:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877049.245653:0:42882:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788877049.245660:0:42882:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877049.245664:0:42882:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x117:0x0](ffffa06198283a88) 00000080:00010000:2.0:1788877049.245668:0:42882:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x117:0x0](ffffa06198283a88) for lock 0x7d14dcdc37048ac4 00000080:00010000:2.0:1788877049.245672:0:42882:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ac4 from it ffffb95ac163bcb8 00010000:00010000:2.0:1788877049.245676:0:42882:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877049.245684:0:42882:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877049.245739:0:42882:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877049.245747:0:42882:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877049.245755:0:42882:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877049.245763:0:42882:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788877049.245770:0:42882:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877049.245775:0:42882:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877049.245777:0:42882:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048abd 00000080:00010000:2.0:1788877049.245782:0:42882:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048abd from it ffffb95ac163bc10 00010000:00010000:2.0:1788877049.245785:0:42882:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877049.245792:0:42882:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877049.245801:0:42882:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788877049.245823:0:42882:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=fsa-oleg311-client.virtnet, flags=0 00000080:00200000:2.0:1788877049.245828:0:42882:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877049.245835:0:42882:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877049.245841:0:42882:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788877049.245850:0:42882:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877049.245857:0:42882:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877049.245862:0:42882:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877049.245864:0:42882:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048abd 00000080:00010000:2.0:1788877049.245868:0:42882:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048abd from it ffffb95ac163bd68 00010000:00010000:2.0:1788877049.245870:0:42882:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788877049.245878:0:42882:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00200000:2.0:1788877049.245887:0:42882:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:2.0:1788877049.245892:0:42882:0:(namei.c:2351:ll_unlink()) VFS Op:name=fsa-oleg311-client.virtnet, dir=[0x200000007:0x1:0x0](ffffa061981363c8) 00010000:00010000:2.0:1788877049.245913:0:42882:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788877049.245935:0:42882:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877049.245946:0:42882:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x117:0x0]: losing layout lock 00010000:00010000:2.0:1788877049.245976:0:42882:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 1/0,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788877049.245989:0:42882:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788877049.245995:0:42882:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:2.0:1788877049.245998:0:42882:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180030400/0x7d14dcdc37048ac4 lrc: 0/0,0 mode: --/PR res: [0x200001b71:0x117:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xf5bf0f7e313b0a6a expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1788877049.246016:0:42882:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031400/0x7d14dcdc37048abd lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0a4e expref: -99 pid: 42882 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000020:00100000:2.0:1788877049.246036:0:42882:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 36, max 7 00000100:00100000:2.0:1788877049.246043:0:42882:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888af800 x1875771511324544/t0(0) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 512/624 e 0 to 0 dl 0 ref 1 fl New:QU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877049.246067:0:42882:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:42882:1875771511324544:192.168.203.111@tcp:36:rm.0 00000100:00100000:2.0:1788877049.246140:0:42882:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cffcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877049.263319:0:42882:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af800 pname:cluuid:pid:xid:nid:opc:job rm:f0204769-56bd-4f5c-ab97-74c450c9e072:42882:1875771511324544:192.168.203.111@tcp:36:rm.0 00000080:00200000:2.0:1788877049.263379:0:42882:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x117:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:2.0:1788877049.263443:0:42882:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x117:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788877049.263451:0:42882:0:(llite_lib.c:2066:ll_clear_inode()) VFS Op:inode=[0x200001b71:0x117:0x0](ffffa06198283a88) 00010000:00010000:2.0:1788877049.597791:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877050.621927:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877051.645801:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877052.030791:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511324800:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.030877:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511325056:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.030907:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:1.0:1788877052.030914:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511324672:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.030940:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511324928:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.030958:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511325184:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.041726:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511324672:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.041750:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511324928:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.041761:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511325184:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.041775:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511324800:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877052.041790:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511325056:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788877052.669632:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:1.0:1788877053.053579:0:29258:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:1.0:1788877053.053596:0:29258:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:1.0:1788877053.053625:0:29258:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### extent: 00000000fd1a227c ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:1.0:1788877053.053645:0:29258:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 00000000fd1a227c ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:1.0:1788877053.053707:0:29258:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000008:00100000:1.0:1788877053.053798:0:29258:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa0618869a680 - object 0x2c0000401:770 offset 0<>4 00000100:00100000:1.0:1788877053.057835:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511325440:192.168.203.111@tcp:16:kworker.0 00000080:00200000:2.0:1788877053.058520:0:29258:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:3.0:1788877053.693670:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877054.717803:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877055.741613:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1788877056.538234:0:42949:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: local REPLAY BARRIER on lustre-MDT0000 00000080:00200000:3.0:1788877056.550448:0:42718:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877056.550472:0:42718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877056.550530:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.550540:0:42718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.550550:0:42718:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877056.550572:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511325568:192.168.203.111@tcp:101:bash.0 00000100:00100000:3.0:1788877056.550641:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168300 going to sleep for 11 seconds 00000100:00100000:3.0:1788877056.554150:0:42718:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511325568:192.168.203.111@tcp:101:bash.0 00010000:00010000:3.0:1788877056.554176:0:42718:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554187:0:42718:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554210:0:42718:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554222:0:42718:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554235:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554242:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554256:0:42718:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877056.554268:0:42718:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188870e00 x1875771511325568/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877072 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788877056.554281:0:42718:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554293:0:42718:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788877056.554299:0:42718:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788877056.554334:0:42718:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788877056.554341:0:42718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048acb 00000080:00010000:3.0:1788877056.554345:0:42718:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877056.554349:0:42718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048acb from it ffffb95ac1823bf8 00010000:00010000:3.0:1788877056.554352:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554370:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877056.554412:0:42718:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877056.554433:0:42718:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877056.554445:0:42718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877056.554453:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877056.554468:0:42718:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877056.554475:0:42718:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877056.554480:0:42718:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877056.554483:0:42718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048acb 00000080:00010000:3.0:1788877056.554487:0:42718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048acb from it ffffb95ac1823b38 00010000:00010000:3.0:1788877056.554490:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877056.554497:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788877056.554505:0:42718:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 3 00000080:00200000:3.0:1788877056.554511:0:42718:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa0619941d500, open_flags 8241, mode 81b6 opened 0 00000080:00200000:3.0:1788877056.554519:0:42718:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open|creat 00000002:00010000:3.0:1788877056.554539:0:42718:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f11.replay-single,[0x200001b71:0x118:0x0]) in obj [0x200000007:0x1:0x0], intent: open|creat flags 0101102 00010000:00010000:3.0:1788877056.554549:0:42718:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.554589:0:42718:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.554595:0:42718:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788877056.554598:0:42718:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048acb lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xf5bf0f7e313b0a94 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.554632:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.554640:0:42718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1788877056.554650:0:42718:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1788877056.554657:0:42718:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188870e00 x1875771511325824/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1788877056.554666:0:42718:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 3/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42718 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877056.554683:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511325824:192.168.203.111@tcp:101:bash.0 00000100:00100000:3.0:1788877056.554806:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168300 going to sleep for 11 seconds 00000100:00100000:3.0:1788877056.565673:0:42718:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870e00 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511325824:192.168.203.111@tcp:101:bash.0 00010000:00010000:3.0:1788877056.565700:0:42718:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 4/0,1 mode: --/CW res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565727:0:42718:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 4/0,1 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565736:0:42718:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 4/0,1 mode: --/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565747:0:42718:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 4/0,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565762:0:42718:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 4/0,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565775:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 5/1,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565784:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 5/1,1 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565792:0:42718:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877056.565804:0:42718:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffffa06188870e00 x1875771511325824/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1788877056.565817:0:42718:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188870e00 x1875771511325824/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788877056.565830:0:42718:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1788877056.565843:0:42718:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.565853:0:42718:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/280 (0 0) 00000080:00200000:3.0:1788877056.565859:0:42718:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:3.0:1788877056.565871:0:42718:0:(namei.c:1628:ll_create_it()) VFS Op:name=f11.replay-single, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open|creat 00000080:00200000:3.0:1788877056.565902:0:42718:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200001b71:0x118:0x0](ffffa061982800c8) 00000080:00200000:3.0:1788877056.565977:0:42718:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x118:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:3.0:1788877056.565982:0:42718:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x200001b71:0x118:0x0](ffffa061982800c8) 00000080:00010000:3.0:1788877056.565985:0:42718:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffffa061982800c8) for lock 0x7d14dcdc37048ad2 00000080:00010000:3.0:1788877056.565995:0:42718:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x118:0x0](ffffa061982800c8) 00000080:00200008:3.0:1788877056.566008:0:42718:0:(file.c:1090:ll_file_open()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffffa061982800c8), flags 101101 00000002:00100000:3.0:1788877056.566027:0:42718:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188870e00 x1875771511325824/t94489280522(94489280522) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'bash.0' uid:0 gid:0 projid:0 00000080:00010000:3.0:1788877056.566056:0:42718:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ad2 from it ffffa061887b8d20 00010000:00010000:3.0:1788877056.566060:0:42718:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.566068:0:42718:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877056.566088:0:42718:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x118:0x0](ffffa061982800c8), inode mode 81a4 mask 40 00000080:00200008:3.0:1788877056.566159:0:42718:0:(file.c:2650:do_file_write_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 0, count: 4 00000080:00200000:3.0:1788877056.566170:0:42718:0:(rw26.c:676:ll_write_begin()) Writing 0 of 0 to 4 bytes 00000080:00200000:3.0:1788877056.566175:0:42718:0:(file.c:2626:ll_do_tiny_write()) result: 0, original count 4 00000080:00200000:3.0:1788877056.566180:0:42718:0:(file.c:2090:ll_file_io_generic()) f11.replay-single: write ppos: 0, bytes: 4 00000080:00200000:3.0:1788877056.566184:0:42718:0:(file.c:1848:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:3.0:1788877056.566190:0:42718:0:(cl_io.c:214:cl_io_rw_init()) header@00000000d69aa72c[0x110, 1, [0x200001b71:0x118:0x0] hash] 00000020:00200000:3.0:1788877056.566192:0:42718:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 4) 0 0 00000080:00200000:3.0:1788877056.566196:0:42718:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788877056.566221:0:42718:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 0] 00020000:00200000:3.0:1788877056.566228:0:42718:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 4) 4, 4 00020000:00200000:3.0:1788877056.566232:0:42718:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788877056.566246:0:42718:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 0] 00000008:00010000:3.0:1788877056.566257:0:42718:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788877056.566261:0:42718:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d7b0, osc lock ffffa061885be4c0, flags 0x0 00010000:00010000:3.0:1788877056.566301:0:42718:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.566310:0:42718:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.566320:0:42718:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: --/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->4095] (req 0->4095) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877056.566348:0:42718:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job bash:f0204769-56bd-4f5c-ab97-74c450c9e072:42718:1875771511325952:192.168.203.111@tcp:101:bash.0 00000100:00100000:3.0:1788877056.566447:0:42718:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168180 going to sleep for 11 seconds 00000080:00200000:3.0:1788877056.588611:0:42950:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:3.0:1788877056.588641:0:42950:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877056.588700:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 42950 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.588712:0:42950:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42950 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.588722:0:42950:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 42950 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877056.588749:0:42950:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42950:1875771511326336:192.168.203.111@tcp:101:grep.0 00000100:00100000:3.0:1788877056.588940:0:42950:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168f00 going to sleep for 11 seconds 00000100:00100000:3.0:1788877056.606054:0:42950:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42950:1875771511326336:192.168.203.111@tcp:101:grep.0 00010000:00010000:3.0:1788877056.606079:0:42950:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606091:0:42950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606103:0:42950:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606115:0:42950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606127:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606143:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606150:0:42950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877056.606162:0:42950:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffffa06188871180 x1875771511326336/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/688 e 0 to 0 dl 1788877072 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788877056.606174:0:42950:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606186:0:42950:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1788877056.606192:0:42950:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788877056.606216:0:42950:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 20480 00000080:00010000:3.0:1788877056.606222:0:42950:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ae0 00000080:00010000:3.0:1788877056.606226:0:42950:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:3.0:1788877056.606229:0:42950:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ae0 from it ffffb95ac1c87bf8 00010000:00010000:3.0:1788877056.606233:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606241:0:42950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877056.606263:0:42950:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:3.0:1788877056.606273:0:42950:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=f11.replay-single, flags=265 00000080:00200000:3.0:1788877056.606279:0:42950:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x118:0x0](ffffa061982800c8), inode mode 81a4 mask 44 00000080:00200008:3.0:1788877056.606287:0:42950:0:(file.c:1090:ll_file_open()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffffa061982800c8), flags 100400 00000002:00010000:3.0:1788877056.606306:0:42950:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x118:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 04240100401 00010000:00010000:3.0:1788877056.606355:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 42950 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.606371:0:42950:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42950 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:3.0:1788877056.606424:0:42950:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:3.0:1788877056.606431:0:42950:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188871180 x1875771511326592/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:3.0:1788877056.606439:0:42950:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42950 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877056.606457:0:42950:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42950:1875771511326592:192.168.203.111@tcp:101:grep.0 00000100:00100000:3.0:1788877056.606644:0:42950:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168f00 going to sleep for 11 seconds 00000100:00100000:3.0:1788877056.611600:0:42950:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871180 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42950:1875771511326592:192.168.203.111@tcp:101:grep.0 00010000:00010000:3.0:1788877056.611634:0:42950:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x118:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.611652:0:42950:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 4/1,0 mode: --/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.611664:0:42950:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.611678:0:42950:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 4/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1788877056.611692:0:42950:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x200002b, status=0 req@ffffa06188871180 x1875771511326592/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:3.0:1788877056.611707:0:42950:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188871180 x1875771511326592/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 648/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:3.0:1788877056.611721:0:42950:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:3.0:1788877056.611735:0:42950:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.611745:0:42950:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 16 res 8589941617/280 (0 0) 00000080:00200000:3.0:1788877056.611752:0:42950:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1788877056.611807:0:42950:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x118:0x0]: layout version change: 0 -> 0 00000080:00010000:3.0:1788877056.611812:0:42950:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffffa061982800c8) for lock 0x7d14dcdc37048ae7 00000080:00010000:3.0:1788877056.611820:0:42950:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ae7 from it ffffb95ac1c87bf0 00010000:00010000:3.0:1788877056.611824:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 3/1,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877056.611835:0:42950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000002:00100000:3.0:1788877056.611866:0:42950:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa06188871180 x1875771511326592/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00200008:3.0:1788877056.611943:0:42950:0:(file.c:6296:ll_getattr_dentry()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffffa061982800c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1788877056.611950:0:42950:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x118:0x0](ffffa061982800c8),name=f11.replay-single 00000002:00010000:3.0:1788877056.611970:0:42950:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x118:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1788877056.611981:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877056.611992:0:42950:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:3.0:1788877056.612000:0:42950:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1788877056.612005:0:42950:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x118:0x0](ffffa061982800c8) 00000080:00010000:3.0:1788877056.612009:0:42950:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x118:0x0](ffffa061982800c8) for lock 0x7d14dcdc37048ad2 00000080:00010000:3.0:1788877056.612013:0:42950:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ad2 from it ffffb95ac1c87cd8 00010000:00010000:3.0:1788877056.612016:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877056.612025:0:42950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887a000/0x7d14dcdc37048ad2 lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x118:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0ab0 expref: -99 pid: 42718 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788877056.612043:0:42950:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:3.0:1788877056.612053:0:42950:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:3.0:1788877056.612083:0:42950:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788877056.612087:0:42950:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9deb0, osc lock ffffa061885be000, flags 0x400001002 00010000:00010000:3.0:1788877056.612103:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877056.612116:0:42950:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00200000:3.0:1788877056.612127:0:42950:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788877056.612131:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 6/0,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1788877056.612140:0:42950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 5/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:3.0:1788877056.612150:0:42950:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788877056.612159:0:42950:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 0 00000020:00010000:3.0:1788877056.612165:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788877056.612169:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612172:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9deb0 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612176:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788877056.612179:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9deb0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788877056.612189:0:42950:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:3.0:1788877056.612215:0:42950:0:(file.c:2483:do_file_read_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 0, count: 32768 00000080:00200000:3.0:1788877056.612261:0:42950:0:(file.c:2090:ll_file_io_generic()) f11.replay-single: read ppos: 4, bytes: 32764 00000080:00200000:3.0:1788877056.612266:0:42950:0:(file.c:1848:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:3.0:1788877056.612272:0:42950:0:(cl_io.c:214:cl_io_rw_init()) header@00000000d69aa72c[0x110, 2, [0x200001b71:0x118:0x0] hash] 00000020:00200000:3.0:1788877056.612275:0:42950:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32768) 0 0 00000080:00200000:3.0:1788877056.612277:0:42950:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788877056.612289:0:42950:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32768) 32768, 32764 00020000:00200000:3.0:1788877056.612294:0:42950:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788877056.612305:0:42950:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:3.0:1788877056.612314:0:42950:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788877056.612317:0:42950:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9deb0, osc lock ffffa061885be000, flags 0x0 00010000:00010000:3.0:1788877056.612327:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.612337:0:42950:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.612348:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.612357:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.612375:0:42950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 6/1,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000080:00200000:3.0:1788877056.612426:0:42950:0:(vvp_io.c:835:vvp_io_read_start()) f11.replay-single: read [4, 32768) 00000001:00010000:3.0:1788877056.612436:0:42950:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:3.0:1788877056.612441:0:42950:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788877056.612444:0:42950:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d0b0, osc lock ffffa061885be130, flags 0x400001002 00010000:00010000:3.0:1788877056.612451:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 5/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.612461:0:42950:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 5/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:3.0:1788877056.612470:0:42950:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788877056.612473:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 8/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.612482:0:42950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 7/1,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000020:00200000:3.0:1788877056.612489:0:42950:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788877056.612494:0:42950:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 0 00000020:00010000:3.0:1788877056.612497:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788877056.612500:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612503:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d0b0 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612506:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788877056.612509:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d0b0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788877056.612515:0:42950:0:(vvp_io.c:893:vvp_io_read_start()) count:32764 iocb pos:4 00000020:00010000:3.0:1788877056.612524:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c54f38 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788877056.612527:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c54f38 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612530:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9deb0 (ffffa061890323f8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788877056.612534:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 5/1,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00010000:00010000:3.0:1788877056.612542:0:42950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00010000:3.0:1788877056.612552:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c54f38 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788877056.612555:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9deb0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788877056.612561:0:42950:0:(vvp_io.c:533:vvp_io_advance()) advancing 0 bytes 00000080:00200000:3.0:1788877056.612567:0:42950:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788877056.612571:0:42950:0:(file.c:2238:ll_file_io_generic()) f11.replay-single: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:3.0:1788877056.612574:0:42950:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:3.0:1788877056.612611:0:42950:0:(file.c:2483:do_file_read_iter()) START file f11.replay-single:[0x200001b71:0x118:0x0], ppos: 4, count: 32768 00000080:00200000:3.0:1788877056.612617:0:42950:0:(file.c:2090:ll_file_io_generic()) f11.replay-single: read ppos: 4, bytes: 32768 00000080:00200000:3.0:1788877056.612620:0:42950:0:(file.c:1848:ll_io_set_mirror()) f11.replay-single: desiginated mirror: 0 00000020:00200000:3.0:1788877056.612624:0:42950:0:(cl_io.c:214:cl_io_rw_init()) header@00000000d69aa72c[0x110, 2, [0x200001b71:0x118:0x0] hash] 00000020:00200000:3.0:1788877056.612626:0:42950:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32772) 0 0 00000080:00200000:3.0:1788877056.612629:0:42950:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788877056.612636:0:42950:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32772) 32772, 32768 00020000:00200000:3.0:1788877056.612639:0:42950:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788877056.612644:0:42950:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 8] 00000008:00010000:3.0:1788877056.612650:0:42950:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1788877056.612652:0:42950:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9deb0, osc lock ffffa061885be000, flags 0x0 00010000:00010000:3.0:1788877056.612660:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877056.612669:0:42950:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 36863) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877056.612679:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877056.612688:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 7/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877056.612696:0:42950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 6/1,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000080:00200000:3.0:1788877056.612709:0:42950:0:(vvp_io.c:835:vvp_io_read_start()) f11.replay-single: read [4, 32772) 00000001:00010000:3.0:1788877056.612717:0:42950:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x118:0x0] 00000008:00010000:3.0:1788877056.612722:0:42950:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:3.0:1788877056.612724:0:42950:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190e9d0b0, osc lock ffffa061885be130, flags 0x400001002 00010000:00010000:3.0:1788877056.612729:0:42950:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 5/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877056.612739:0:42950:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 5/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:3.0:1788877056.612747:0:42950:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:3.0:1788877056.612750:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 8/1,1 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877056.612758:0:42950:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 7/1,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:3.0:1788877056.612765:0:42950:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1788877056.612769:0:42950:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x118:0x0] updating i_size 4 i_blocks 0 00000020:00010000:3.0:1788877056.612772:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788877056.612774:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612777:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9d0b0 (ffffa061890323f8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612779:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615d4d0 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788877056.612782:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9d0b0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788877056.612786:0:42950:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:4 00000020:00010000:3.0:1788877056.612790:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c54f38 (ffffa061890322d8/0) at cl_lock_release():245 00000020:00010000:3.0:1788877056.612793:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c54f38 (ffffa061890322d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1788877056.612795:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190e9deb0 (ffffa061890323f8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1788877056.612798:0:42950:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 5/1,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1788877056.612807:0:42950:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 4/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00010000:3.0:1788877056.612815:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c54f38 (ffffa061890322d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1788877056.612818:0:42950:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190e9deb0 (ffffa061890322d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1788877056.612822:0:42950:0:(vvp_io.c:533:vvp_io_advance()) advancing 0 bytes 00000080:00200000:3.0:1788877056.612826:0:42950:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788877056.612830:0:42950:0:(file.c:2238:ll_file_io_generic()) f11.replay-single: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:3.0:1788877056.612832:0:42950:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:3.0:1788877056.612845:0:42950:0:(file.c:479:ll_file_release()) START file f11.replay-single:[0x200001b71:0x118:0x0](ffffa061982800c8), flags 100000 00010000:00010000:3.0:1788877056.612859:0:42950:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877056.630286:0:42951:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8),name=/ 00000002:00010000:2.0:1788877056.630313:0:42951:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877056.630343:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877056.630368:0:42951:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788877056.630378:0:42951:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877056.630590:0:42951:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) 00000080:00010000:2.0:1788877056.630595:0:42951:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffffa061981363c8) for lock 0x7d14dcdc37048ae0 00000080:00010000:2.0:1788877056.630603:0:42951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048ae0 from it ffffb95ac1d1fbf8 00010000:00010000:2.0:1788877056.630607:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877056.630619:0:42951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877056.630635:0:42951:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffffa061981363c8), inode mode 41ed mask 1 00000080:00200000:2.0:1788877056.630648:0:42951:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffffa061981363c8), file ffffa06198ade480, open_flags 8100, mode 0 opened 0 00000080:00200000:2.0:1788877056.630655:0:42951:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=f11.replay-single-2, dir=[0x200000007:0x1:0x0](ffffa061981363c8), intent=open 00000002:00010000:2.0:1788877056.630669:0:42951:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: f11.replay-single-2,[0x0:0x0:0x0]) in obj [0x200000007:0x1:0x0], intent: open flags 0100401 00010000:00010000:2.0:1788877056.630716:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 42951 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.630723:0:42951:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42951 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1788877056.630735:0:42951:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1788877056.630747:0:42951:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ace00 x1875771511326848/t0(0) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1788877056.630757:0:42951:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT flags: 0x0 pid: 42951 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1788877056.630776:0:42951:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42951:1875771511326848:192.168.203.111@tcp:101:grep.0 00000100:00100000:2.0:1788877056.631026:0:42951:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cffcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877056.647481:0:42951:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42951:1875771511326848:192.168.203.111@tcp:101:grep.0 00010000:00010000:2.0:1788877056.647509:0:42951:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647526:0:42951:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647534:0:42951:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 4/1,0 mode: --/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647545:0:42951:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647556:0:42951:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647570:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647578:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 5/2,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647585:0:42951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 4/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1788877056.647597:0:42951:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffffa061888ace00 x1875771511326848/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1788877056.647609:0:42951:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061888ace00 x1875771511326848/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 664/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1788877056.647620:0:42951:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 56 ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1788877056.647632:0:42951:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647641:0:42951:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffffa06183d53000 type 13 mode 4 res 8589941617/277 (0 0) 00000080:00200000:2.0:1788877056.647647:0:42951:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000007:0x1:0x0] suppgids 0 -1: rc 0 00000080:00200000:2.0:1788877056.647687:0:42951:0:(namei.c:147:ll_iget()) got inode: [0x200001b71:0x115:0x0](ffffa061981c6c08): rc = 0 00000080:00200000:2.0:1788877056.647697:0:42951:0:(file.c:6843:ll_layout_conf()) [0x200001b71:0x115:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1788877056.647701:0:42951:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffffa061981c6c08) for lock 0x7d14dcdc37048aee 00000080:00010000:2.0:1788877056.647712:0:42951:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffffa061981c6c08) 00000080:00200008:2.0:1788877056.647726:0:42951:0:(file.c:1090:ll_file_open()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffffa061981c6c08), flags 100400 00000002:00100000:2.0:1788877056.647745:0:42951:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffffa061888ace00 x1875771511326848/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788877072 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'grep.0' uid:0 gid:0 projid:0 00000080:00010000:2.0:1788877056.647788:0:42951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aee from it ffffa061888c0600 00010000:00010000:2.0:1788877056.647793:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877056.647808:0:42951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788877056.647829:0:42951:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200001b71:0x115:0x0](ffffa061981c6c08), inode mode 81a4 mask 44 00000080:00200008:2.0:1788877056.647852:0:42951:0:(file.c:6296:ll_getattr_dentry()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffffa061981c6c08), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1788877056.647858:0:42951:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200001b71:0x115:0x0](ffffa061981c6c08),name=f11.replay-single-2 00000002:00010000:2.0:1788877056.647875:0:42951:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200001b71:0x115:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1788877056.647884:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877056.647894:0:42951:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 2/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1788877056.647900:0:42951:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1788877056.647904:0:42951:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200001b71:0x115:0x0](ffffa061981c6c08) 00000080:00010000:2.0:1788877056.647906:0:42951:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200001b71:0x115:0x0](ffffa061981c6c08) for lock 0x7d14dcdc37048aee 00000080:00010000:2.0:1788877056.647910:0:42951:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x7d14dcdc37048aee from it ffffb95ac1d1fcd8 00010000:00010000:2.0:1788877056.647913:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 3/1,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1788877056.647919:0:42951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:2.0:1788877056.647935:0:42951:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 restore needed 0 00000001:00010000:2.0:1788877056.647944:0:42951:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:2.0:1788877056.647961:0:42951:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788877056.647965:0:42951:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190fe7430, osc lock ffffa06187df7560, flags 0x400001002 00010000:00010000:2.0:1788877056.647981:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788877056.647991:0:42951:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00000020:00200000:2.0:1788877056.648002:0:42951:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788877056.648005:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 3 lru_type: 0 00010000:00010000:2.0:1788877056.648012:0:42951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1788877056.648023:0:42951:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788877056.648033:0:42951:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 8 00000020:00010000:2.0:1788877056.648037:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:2.0:1788877056.648040:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648043:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190fe7430 (ffffa06189032ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648046:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1788877056.648049:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190fe7430 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1788877056.648058:0:42951:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/1, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200008:2.0:1788877056.648073:0:42951:0:(file.c:2483:do_file_read_iter()) START file f11.replay-single-2:[0x200001b71:0x115:0x0], ppos: 0, count: 32768 00000080:00200000:2.0:1788877056.648114:0:42951:0:(file.c:2090:ll_file_io_generic()) f11.replay-single-2: read ppos: 4, bytes: 32764 00000080:00200000:2.0:1788877056.648119:0:42951:0:(file.c:1848:ll_io_set_mirror()) f11.replay-single-2: desiginated mirror: 0 00000020:00200000:2.0:1788877056.648125:0:42951:0:(cl_io.c:214:cl_io_rw_init()) header@000000003825996c[0x110, 2, [0x200001b71:0x115:0x0] hash] 00000020:00200000:2.0:1788877056.648127:0:42951:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32768) 0 0 00000080:00200000:2.0:1788877056.648142:0:42951:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788877056.648154:0:42951:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32768) 32768, 32764 00020000:00200000:2.0:1788877056.648158:0:42951:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788877056.648169:0:42951:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 7] 00000008:00010000:2.0:1788877056.648177:0:42951:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788877056.648180:0:42951:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190fe7430, osc lock ffffa06187df7560, flags 0x0 00010000:00010000:2.0:1788877056.648188:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788877056.648197:0:42951:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 32767) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788877056.648205:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788877056.648212:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788877056.648219:0:42951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 5/1,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000080:00200000:2.0:1788877056.648226:0:42951:0:(vvp_io.c:835:vvp_io_read_start()) f11.replay-single-2: read [4, 32768) 00000001:00010000:2.0:1788877056.648235:0:42951:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:2.0:1788877056.648238:0:42951:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788877056.648240:0:42951:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190fe7a30, osc lock ffffa06187df6d10, flags 0x400001002 00010000:00010000:2.0:1788877056.648245:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788877056.648253:0:42951:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1788877056.648260:0:42951:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788877056.648263:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 7/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788877056.648270:0:42951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/1,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000020:00200000:2.0:1788877056.648275:0:42951:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788877056.648279:0:42951:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 8 00000020:00010000:2.0:1788877056.648281:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:2.0:1788877056.648284:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648286:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190fe7a30 (ffffa06189032ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648289:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1788877056.648292:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190fe7a30 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1788877056.648296:0:42951:0:(vvp_io.c:893:vvp_io_read_start()) count:32764 iocb pos:4 00000020:00010000:2.0:1788877056.648304:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c544e8 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:2.0:1788877056.648306:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c544e8 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648308:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190fe7430 (ffffa06189032ab8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788877056.648311:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:2.0:1788877056.648319:0:42951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00010000:2.0:1788877056.648327:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c544e8 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1788877056.648330:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190fe7430 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1788877056.648334:0:42951:0:(vvp_io.c:533:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1788877056.648341:0:42951:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788877056.648344:0:42951:0:(file.c:2238:ll_file_io_generic()) f11.replay-single-2: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1788877056.648347:0:42951:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1788877056.648414:0:42951:0:(file.c:2483:do_file_read_iter()) START file f11.replay-single-2:[0x200001b71:0x115:0x0], ppos: 4, count: 32768 00000080:00200000:2.0:1788877056.648421:0:42951:0:(file.c:2090:ll_file_io_generic()) f11.replay-single-2: read ppos: 4, bytes: 32768 00000080:00200000:2.0:1788877056.648423:0:42951:0:(file.c:1848:ll_io_set_mirror()) f11.replay-single-2: desiginated mirror: 0 00000020:00200000:2.0:1788877056.648427:0:42951:0:(cl_io.c:214:cl_io_rw_init()) header@000000003825996c[0x110, 2, [0x200001b71:0x115:0x0] hash] 00000020:00200000:2.0:1788877056.648429:0:42951:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [4, 32772) 0 0 00000080:00200000:2.0:1788877056.648431:0:42951:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1788877056.648437:0:42951:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 4 chunk: [4, 32772) 32772, 32768 00020000:00200000:2.0:1788877056.648439:0:42951:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1788877056.648444:0:42951:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 8] 00000008:00010000:2.0:1788877056.648450:0:42951:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1788877056.648452:0:42951:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190fe7430, osc lock ffffa06187df7560, flags 0x0 00010000:00010000:2.0:1788877056.648459:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788877056.648468:0:42951:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 36863) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 2/0,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788877056.648476:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788877056.648485:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788877056.648491:0:42951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 5/1,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:2.0:1788877056.648498:0:42951:0:(vvp_io.c:835:vvp_io_read_start()) f11.replay-single-2: read [4, 32772) 00000001:00010000:2.0:1788877056.648503:0:42951:0:(glimpse.c:73:cl_glimpse_lock()) Glimpsing inode [0x200001b71:0x115:0x0] 00000008:00010000:2.0:1788877056.648507:0:42951:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: a 00000008:00010000:2.0:1788877056.648509:0:42951:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffffa06190fe7a30, osc lock ffffa06187df6d10, flags 0x400001002 00010000:00010000:2.0:1788877056.648513:0:42951:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788877056.648521:0:42951:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 18446744073709551615) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1788877056.648529:0:42951:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00010000:00010000:2.0:1788877056.648532:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 7/1,1 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788877056.648539:0:42951:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 6/1,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000020:00200000:2.0:1788877056.648545:0:42951:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1788877056.648548:0:42951:0:(file.c:1775:ll_merge_attr_nolock()) [0x200001b71:0x115:0x0] updating i_size 4 i_blocks 8 00000020:00010000:2.0:1788877056.648550:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:2.0:1788877056.648553:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648555:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190fe7a30 (ffffa06189032ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648557:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa0618615c250 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1788877056.648559:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190fe7a30 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1788877056.648563:0:42951:0:(vvp_io.c:893:vvp_io_read_start()) count:32768 iocb pos:4 00000020:00010000:2.0:1788877056.648567:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffffa06191c544e8 (ffffa06189032d88/0) at cl_lock_release():245 00000020:00010000:2.0:1788877056.648570:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06191c544e8 (ffffa06189032d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1788877056.648572:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffffa06190fe7430 (ffffa06189032ab8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1788877056.648576:0:42951:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 4/1,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x810020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:2.0:1788877056.648584:0:42951:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffffa06189393000 lock: 00000000fadb0c3d/0x7d14dcdc37048a9a lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) gid 0 flags: 0x800020080000000 nid: local remote: 0xf5bf0f7e313b09c9 expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 6 lru_type: 0 00000020:00010000:2.0:1788877056.648592:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06191c544e8 (ffffa06189032d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1788877056.648594:0:42951:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffffa06190fe7430 (ffffa06189032d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1788877056.648598:0:42951:0:(vvp_io.c:533:vvp_io_advance()) advancing 0 bytes 00000080:00200000:2.0:1788877056.648602:0:42951:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1788877056.648605:0:42951:0:(file.c:2238:ll_file_io_generic()) f11.replay-single-2: 1 io complete with rc: 0, result: 0, restart: 0 00000080:00200000:2.0:1788877056.648607:0:42951:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 0 00000080:00200008:2.0:1788877056.648620:0:42951:0:(file.c:479:ll_file_release()) START file f11.replay-single-2:[0x200001b71:0x115:0x0](ffffa061981c6c08), flags 100000 00000002:00100000:2.0:1788877056.648657:0:42951:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa061888ace00 x1875771511326848/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788877072 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1788877056.648667:0:42951:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1788877056.648672:0:42951:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa061888ad180 x1875771511327104/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1788877056.648690:0:42951:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42951:1875771511327104:192.168.203.111@tcp:35:grep.0 00000100:00100000:2.0:1788877056.648751:0:42951:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06187cffcc0 going to sleep for 11 seconds 00000100:00100000:2.0:1788877056.652534:0:42951:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job grep:f0204769-56bd-4f5c-ab97-74c450c9e072:42951:1875771511327104:192.168.203.111@tcp:35:grep.0 00000002:00100000:2.0:1788877056.652569:0:42951:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffffa061888ace00 x1875771511326848/t94489280525(94489280525) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 592/664 e 0 to 0 dl 1788877072 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1788877056.770299:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788877057.149851:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511327360:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877057.149982:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1788877057.149991:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511327232:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877057.150024:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511327488:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877057.152753:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511327232:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877057.152785:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511327488:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877057.154708:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511327360:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877057.796896:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1788877058.173698:0:29258:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788877058.173714:0:29258:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1788877058.173738:0:29258:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x115:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1788877058.173748:0:29258:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:3.0:1788877058.173753:0:29258:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000008:00010000:3.0:1788877058.173770:0:29258:0:(osc_cache.c:2074:try_to_add_extent_for_io()) ### extent: 0000000022c64907 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00000008:00010000:3.0:1788877058.173789:0:29258:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 0000000022c64907 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1788877058.173842:0:29258:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 4095) ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00100000:3.0:1788877058.173938:0:29258:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffffa06188870a80 - object 0x280000401:770 offset 0<>4 00000100:00100000:3.0:1788877058.174010:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511327744:192.168.203.111@tcp:16:kworker.0 00000100:00100000:3.0:1788877058.174111:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1788877058.174118:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511327616:192.168.203.111@tcp:4:grep.0 00000008:00010000:3.0:1788877058.181069:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 0000000022c64907 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000008:00010000:3.0:1788877058.181115:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 0000000022c64907 ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00100000:3.0:1788877058.181148:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511327616:192.168.203.111@tcp:4:grep.0 00000100:00100000:3.0:1788877058.183800:0:2397:0:(client.c:3126:ptlrpc_free_committed()) lustre-OST0000-osc-ffffa06189393000: committing for last_committed 8589934863 gen 5 00000100:00100000:3.0:1788877058.183850:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511327744:192.168.203.111@tcp:16:kworker.0 00000080:00200000:1.0:1788877058.183921:0:29258:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1788877058.814685:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877059.837519:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877060.863621:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877061.885745:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1788877062.271166:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511327872:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877062.271571:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328128:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877062.271605:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328384:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877062.271637:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788877062.271644:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328000:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877062.271667:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328256:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788877062.279653:0:2398:0:(recover.c:205:ptlrpc_request_handle_notconn()) import lustre-MDT0000-mdc-ffffa06189393000 of lustre-MDT0000_UUID@192.168.203.111@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1788877062.279664:0:2398:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1788877062.279670:0:2398:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffffa06189393000: Connection to lustre-MDT0000 (at 192.168.203.111@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1788877062.279678:0:2398:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877062.279684:0:2398:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2013 00000100:00080000:3.0:1788877062.279696:0:2398:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877062.279710:0:2398:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877062.279747:0:2398:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511328512/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877062.279787:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328000:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877062.279809:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328256:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877062.279822:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328128:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877062.279834:0:2398:0:(client.c:3126:ptlrpc_free_committed()) lustre-OST0001-osc-ffffa06189393000: committing for last_committed 4294967572 gen 5 00000100:00100000:3.0:1788877062.279851:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328384:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877062.310532:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877062.310550:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788877062.310556:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511328512:192.168.203.111@tcp:38:ptlrpcd_01_01.0 00010000:00010000:3.0:1788877062.909608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877063.938509:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877064.958999:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877065.982283:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877067.007165:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877067.389616:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877067.389625:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877067.389629:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877067.389643:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2054 00000100:00080000:3.0:1788877067.389656:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788877067.389665:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877067.389678:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877067.389698:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511328768/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877067.389790:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328896:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877067.389903:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511329152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877067.390194:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511328768:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877067.390377:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511328640:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877067.390463:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329024:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877067.400776:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329024:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877067.401072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877067.401079:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788877067.401084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511328768:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877067.407124:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511329152:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877067.407175:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511328896:192.168.203.111@tcp:400:kworker.0 00010000:00010000:2.0:1788877068.030863:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877069.053585:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877070.077787:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877071.105680:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877072.125720:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788877072.510000:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511329280:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788877072.510142:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877072.510148:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877072.510151:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877072.510159:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2059 00000100:00080000:3.0:1788877072.510168:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1788877072.510175:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877072.510189:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877072.510208:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511329408/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877072.511890:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1788877072.511921:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329536:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877072.512164:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511329408:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877072.514863:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329664:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877072.514950:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1788877072.514956:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329792:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877072.525490:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877072.525507:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788877072.525509:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329792:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877072.525515:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511329408:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877072.525672:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329664:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877072.525686:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511329536:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877073.150338:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877074.174600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877075.198338:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877076.221683:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877077.246251:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877077.631104:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2396:1875771511329920:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877077.631410:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511330048:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877077.631518:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788877077.631526:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511330176:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877077.632014:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511330432:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877077.632055:0:2395:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1788877077.632062:0:2395:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511330304:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788877077.636941:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877077.636948:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788877077.636952:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511330048:192.168.203.111@tcp:38:kworker.0 00010000:00010000:3.0:1788877078.269565:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1788877078.525619:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877062/real 1788877062] req@ffffa06188870380 x1875771511327872/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877078 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877078.525662:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06188870380 x1875771511327872/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877078 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877078.525673:0:2398:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1788877078.525677:0:2398:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.111@tcp: Connection to MGS (at 192.168.203.111@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1788877078.525681:0:2398:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.111@tcp for MGC192.168.203.111@tcp not replayable, auto-deactivating 00000100:00080000:2.0:1788877078.525684:0:2398:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:2.0:1788877078.525688:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188870380 x1875771511327872/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877078 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877078.525698:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188872680 x1875771511328640/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877083 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877078.525728:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188873480 x1875771511329280/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877088 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877078.525738:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06198aff100 x1875771511329920/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877093 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1788877078.525756:0:2398:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1788877078.525759:0:2398:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1788877078.525761:0:2398:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:2.0:1788877078.525763:0:2398:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:2.0:1788877078.525802:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511327872:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877078.525948:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511329280:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877078.525963:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511328640:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877078.526005:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877078.526011:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877078.526014:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877078.526021:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2013 00000100:00080000:1.0:1788877078.526035:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877078.526074:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511330560/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877078.526146:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511330560:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877078.526542:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877078.526547:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877078.526549:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877078.526554:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2069 00000100:00080000:1.0:1788877078.526559:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877078.526564:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877078.526575:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877078.526589:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511330688/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877078.526716:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511330688:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877078.526889:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511330816:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877078.526921:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511331072:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877078.526943:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788877078.526949:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511330944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877078.538434:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511330944:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877078.538457:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511330816:192.168.203.111@tcp:400:kworker.0 00000100:00100000:1.0:1788877078.538468:0:2396:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511331072:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788877078.540408:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877078.540423:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:2.0:1788877078.540429:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511330688:192.168.203.111@tcp:38:kworker.0 00010000:00010000:1.0:1788877079.293863:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877080.318045:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877081.341593:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877082.365630:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877083.389790:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877083.773690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877083.773704:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877083.773708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877083.773713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877083.773719:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877083.773737:0:37:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2070 00000100:00080000:2.0:1788877083.773754:0:37:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877083.773764:0:37:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877083.773780:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877083.773831:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511331200/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877083.773910:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511331200:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877083.773978:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511331456:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877083.774080:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:2.0:1788877083.774087:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511331328:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877083.774114:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511331584:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877083.778803:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511331456:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877083.779372:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511331584:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877083.780213:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877083.780225:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-19) 00000100:00100000:1.0:1788877083.780243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511331200:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877083.780327:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511331328:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877084.413837:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877085.437887:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877086.462954:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877087.485578:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877088.510412:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877088.894378:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877088.894413:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877078/real 1788877078] req@ffffa0618869ad80 x1875771511330560/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877088 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877088.894456:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511330560/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877088 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877088.894523:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788877088.894527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877088.894530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877088.894534:0:37:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877088.894542:0:37:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2075 00000100:00080000:1.0:1788877088.894553:0:37:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877088.894561:0:37:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877088.894577:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877088.894615:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511331712/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877088.895378:0:2396:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1788877088.895403:0:2396:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2396:1875771511332096:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877088.897235:0:2395:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 1 blwis in pool 00000100:00100000:1.0:1788877088.897359:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511331840:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877088.899947:0:2395:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 0/0 locks to regular blp list, 0 blwis in pool 00000100:00100000:1.0:1788877088.899966:0:2395:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2395:1875771511331968:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788877088.900337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877088.900364:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877088.900369:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511330560:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877088.900409:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511331712:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788877088.900410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877088.900419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877088.900431:0:37:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877088.900438:0:37:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2070 00000100:00080000:1.0:1788877088.900447:0:37:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 5s 00000100:00080000:1.0:1788877088.900454:0:37:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877088.900485:0:37:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511332224/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877088.900522:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511332224:192.168.203.111@tcp:250:kworker.0 00010000:00010000:3.0:1788877088.900777:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877088.904415:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788877088.904433:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877088.904435:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877088.904437:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877088.910152:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) MGC192.168.203.111@tcp: connect to target with instance 0 10000000:01000000:2.0:1788877088.910173:0:2394:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:02000400:2.0:1788877088.910182:0:2394:0:(import.c:1328:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.203.111@tcp) after server handle changed from 0xf5bf0f7e313b0603 to 0xf5bf0f7e313b0c46 00000100:00080000:2.0:1788877088.910189:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to EVICTED 00000100:00080000:2.0:1788877088.910195:0:2394:0:(import.c:1718:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.203.111@tcp; invalidating 00000100:00100000:1.0:1788877088.911329:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511332224:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877088.911413:0:43219:0:(import.c:1642:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.203.111@tcp to MGS@192.168.203.111@tcp 00000100:00100000:2.0:1788877088.911417:0:43219:0:(import.c:323:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:2.0:1788877088.911421:0:43219:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808003 00010000:00010000:2.0:1788877088.911440:0:43219:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b06d5 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788877088.911453:0:43219:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b06d5 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788877088.911460:0:43219:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1788877088.911482:0:43219:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788877088.911493:0:43219:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887a200/0x7d14dcdc37048a4d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b06d5 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877088.911508:0:43219:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b06e3 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788877088.911515:0:43219:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b06e3 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788877088.911521:0:43219:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:2.0:1788877088.911525:0:43219:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788877088.911528:0:43219:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa06180030200/0x7d14dcdc37048a5b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b06e3 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877088.911544:0:43219:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b06ce expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788877088.911551:0:43219:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b06ce expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788877088.911557:0:43219:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00010000:2.0:1788877088.911560:0:43219:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788877088.911564:0:43219:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618887a800/0x7d14dcdc37048a46 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b06ce expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1788877088.911576:0:43219:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110e400000000 nid: local remote: 0xf5bf0f7e313b06dc expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1788877088.911583:0:43219:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x110f400000000 nid: local remote: 0xf5bf0f7e313b06dc expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1788877088.911589:0:43219:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1788877088.911593:0:43219:0:(ldlm_request.c:1407:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1788877088.911596:0:43219:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.111@tcp lock: ffffa0618902f000/0x7d14dcdc37048a54 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x590f400000000 nid: local remote: 0xf5bf0f7e313b06dc expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:2.0:1788877088.911609:0:43219:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from EVICTED to RECOVER 00000100:00080000:2.0:1788877088.911614:0:43219:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from RECOVER to FULL 10000000:01000000:2.0:1788877088.911616:0:43219:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:2.0:1788877088.911620:0:43219:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:02000000:2.0:1788877088.911623:0:43219:0:(import.c:1787:ptlrpc_import_recovery_state_machine()) MGC192.168.203.111@tcp: Connection restored to 192.168.203.111@tcp (at 192.168.203.111@tcp) 00010000:00080000:2.0:1788877088.918492:0:2394:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffffa06188699f80 x1875771511331712/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/416 e 0 to 0 dl 1788877113 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877088.918524:0:2394:0:(import.c:1176:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffffa06189393000: connect to target with instance 23 00000100:00080000:2.0:1788877088.918547:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to REPLAY 00000100:00080000:2.0:1788877088.918555:0:2394:0:(import.c:1744:ptlrpc_import_recovery_state_machine()) replay requested by lustre-MDT0000_UUID 00000100:00100000:2.0:1788877088.918558:0:2394:0:(client.c:3126:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffffa06189393000: committing for last_committed 94489280519 gen 1 00000100:00100000:2.0:1788877088.918566:0:2394:0:(client.c:3170:ptlrpc_free_committed()) @@@ stopping search req@ffffa06188870000 x1875771511324032/t94489280520(94489280520) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788877065 ref 1 fl Complete:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877088.918574:0:2394:0:(recover.c:53:ptlrpc_replay_next()) import ffffa06183c27000 from lustre-MDT0000_UUID committed 94489280519 last 0 00000100:00080000:2.0:1788877088.918582:0:2394:0:(client.c:3554:ptlrpc_replay_req()) @@@ REPLAY req@ffffa06188870000 x1875771511324032/t94489280520(94489280520) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 0 dl 1788877065 ref 1 fl New:RQU/204/0 rc 0/0 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877088.918597:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511331712:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877088.918629:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511324032:192.168.203.111@tcp:36:mcreate.0 10000000:01000000:3.0:1788877090.493455:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-cliir 10000000:01000000:3.0:1788877090.493463:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffffa06189393000 from 92 10000000:01000000:3.0:1788877090.493468:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1788877090.493513:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887aa00/0x7d14dcdc37048af5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.493526:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887aa00/0x7d14dcdc37048af5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.493538:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887aa00/0x7d14dcdc37048af5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877090.493565:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332352:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.493653:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.497014:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332352:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788877090.497036:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887aa00/0x7d14dcdc37048af5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0d11 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.497058:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887aa00/0x7d14dcdc37048af5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b0d11 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877090.497126:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332480:192.168.203.111@tcp:256:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.497182:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.504732:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332480:192.168.203.111@tcp:256:ll_cfg_requeue.0 10000000:01000000:3.0:1788877090.504812:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1788877090.504822:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887aa00/0x7d14dcdc37048af5 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0d11 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.504833:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887aa00/0x7d14dcdc37048af5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0d11 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788877090.504842:0:15194:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:3.0:1788877090.504845:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffffa06189393000 from 52 10000000:01000000:3.0:1788877090.504848:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:3.0:1788877090.504876:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048afc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.504884:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048afc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.504893:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048afc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877090.504922:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332608:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.505013:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.506917:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332608:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788877090.506937:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048afc lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0d18 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.506948:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048afc lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b0d18 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877090.507000:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332736:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.507086:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.510940:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332736:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.510990:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332864:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.511084:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.514246:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332864:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000020:01000000:3.0:1788877090.515151:0:15194:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 52-51 (rc=0) 10000000:01000000:3.0:1788877090.515168:0:15194:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.111@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1788877090.515178:0:15194:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048afc lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0d18 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.515190:0:15194:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.111@tcp lock: ffffa0618887be00/0x7d14dcdc37048afc lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0d18 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1788877090.515211:0:15194:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:3.0:1788877090.515215:0:15194:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffffa06189393000 from 7 10000000:01000000:3.0:1788877090.515218:0:15194:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1788877090.515594:0:15194:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.111@tcp lock: ffffa0618887b200/0x7d14dcdc37048b03 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.515603:0:15194:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.111@tcp lock: ffffa0618887b200/0x7d14dcdc37048b03 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.515619:0:15194:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.111@tcp lock: ffffa0618887b200/0x7d14dcdc37048b03 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877090.515641:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332992:192.168.203.111@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.515760:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.519928:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511332992:192.168.203.111@tcp:101:ll_cfg_requeue.0 00010000:00010000:3.0:1788877090.519953:0:15194:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.111@tcp lock: ffffa0618887b200/0x7d14dcdc37048b03 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xf5bf0f7e313b0d1f expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1788877090.519965:0:15194:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.111@tcp lock: ffffa0618887b200/0x7d14dcdc37048b03 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xf5bf0f7e313b0d1f expref: -99 pid: 15194 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1788877090.520028:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511333120:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.520104:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.524513:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511333120:192.168.203.111@tcp:501:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.524565:0:15194:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511333248:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1788877090.524640:0:15194:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06189168000 going to sleep for 11 seconds 00000100:00100000:3.0:1788877090.526595:0:15194:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:15194:1875771511333248:192.168.203.111@tcp:503:ll_cfg_requeue.0 00000040:00080000:2.0:1788877090.535468:0:43230:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 64768 count 7 00010000:00010000:3.0:1788877090.557998:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877091.582542:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877093.629546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877094.014078:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level REPLAY/5 force 0 force_next 1 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877094.014085:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: REPLAY) 00000100:00100000:3.0:1788877094.014226:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511333760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877094.014316:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334016:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877094.014346:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:3.0:1788877094.014352:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511333888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877094.014371:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334144:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877094.653563:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877096.701529:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877097.725707:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788877099.134903:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511334272:192.168.203.111@tcp:400:kworker.0 00000100:00080000:3.0:1788877099.136006:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level REPLAY/5 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877099.136012:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: REPLAY) 00000100:00100000:2.0:1788877099.136551:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788877099.136560:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334400:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877099.138830:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334528:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877099.139643:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1788877099.139651:0:2398:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334656:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877099.773524:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877100.798941:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877102.845696:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877103.870182:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1788877104.254301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877088/real 1788877088] req@ffffa06188870000 x1875771511324032/t94489280520(94489280520) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 1 dl 1788877104 ref 2 fl Rpc:XQr/204/ffffffff rc 0/-1 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877104.254353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=REPLAY (now=REPLAY) req@ffffa06188870000 x1875771511324032/t94489280520(94489280520) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 1 dl 1788877104 ref 2 fl Rpc:XQU/204/ffffffff rc 0/-1 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877104.254361:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level REPLAY/5 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877104.254370:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: REPLAY) 00000100:00020000:1.0:1788877104.254379:0:2394:0:(client.c:3399:ptlrpc_replay_interpret()) @@@ request replay timed out req@ffffa06188870000 x1875771511324032/t94489280520(94489280520) o36->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 528/448 e 0 to 1 dl 1788877104 ref 2 fl Interpret:EXQU/204/ffffffff rc -110/-1 job:'mcreate.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877104.254478:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511334912:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877104.254626:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335168:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877104.254672:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788877104.254679:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511334784:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877104.254720:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335040:192.168.203.111@tcp:400:kworker.0 00000100:00080000:1.0:1788877104.278634:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from REPLAY to CONNECTING 00000100:00080000:1.0:1788877104.278649:0:2394:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2080 00000100:00080000:1.0:1788877104.278667:0:2394:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877104.278682:0:2394:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877104.278718:0:2394:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511335296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877104.278737:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511324032:192.168.203.111@tcp:36:mcreate.0 00000100:00100000:1.0:1788877104.278756:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511335296:192.168.203.111@tcp:38:ptlrpcd_rcv.0 00010000:00010000:3.0:1788877105.917538:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877106.942420:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877108.989686:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877109.373655:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877109.373665:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1788877109.373735:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877109.373851:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335808:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877109.373886:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:2.0:1788877109.373892:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511335424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877109.373928:0:2397:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335680:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877110.013586:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1788877110.333544:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877094/real 1788877094] req@ffffa061888aea00 x1875771511333888/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877110.333587:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888aea00 x1875771511333888/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877110.333597:0:2398:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1788877110.333601:0:2398:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffffa06189393000: Connection to lustre-MDT0001 (at 192.168.203.111@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1788877110.333608:0:2398:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00000400:3.0:1788877110.333636:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877094/real 1788877094] req@ffffa061888ad880 x1875771511334144/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877110.333650:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888ad880 x1875771511334144/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877110.333657:0:2398:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1788877110.333663:0:2398:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffffa06189393000: Connection to lustre-OST0001 (at 192.168.203.111@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1788877110.333670:0:2398:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00000400:3.0:1788877110.333678:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877094/real 1788877094] req@ffffa061888ac700 x1875771511333760/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877110.333690:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888ac700 x1875771511333760/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877110.333696:0:2398:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1788877110.333701:0:2398:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.111@tcp: Connection to MGS (at 192.168.203.111@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1788877110.333707:0:2398:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.111@tcp for MGC192.168.203.111@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1788877110.333710:0:2398:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1788877110.333714:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888ac700 x1875771511333760/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877110.333723:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa06188872680 x1875771511334272/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877115 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877110.333733:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888aed80 x1875771511334784/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877120 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877110.333753:0:2398:0:(client.c:3591:ptlrpc_abort_inflight()) @@@ inflight req@ffffa061888ac380 x1875771511335424/t0(0) o400->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 224/224 e 0 to 0 dl 1788877125 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1788877110.333762:0:2398:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1788877110.333767:0:2398:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 00000100:00080000:2.0:1788877110.333769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 10000000:01000000:3.0:1788877110.333770:0:2398:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1788877110.333778:0:2398:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00080000:2.0:1788877110.333790:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877110.333797:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877110.333812:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2080 00000100:00080000:2.0:1788877110.333842:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00000400:3.0:1788877110.333865:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877094/real 1788877094] req@ffffa061888ad180 x1875771511334016/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877110.333882:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888ad180 x1875771511334016/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877110 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877110.333895:0:2398:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1788877110.333899:0:2398:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffffa06189393000: Connection to lustre-OST0000 (at 192.168.203.111@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1788877110.333903:0:2398:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00100000:2.0:1788877110.333922:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511335936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877110.333986:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877110.333991:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877110.333994:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877110.334000:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877110.334002:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877110.334006:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 710 00000100:00080000:2.0:1788877110.334013:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877110.334029:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877110.334044:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511336064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877110.334056:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877110.334059:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877110.334061:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877110.334065:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1029 00000100:00080000:2.0:1788877110.334073:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877110.334078:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877110.334092:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511336192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877110.334100:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877110.334111:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877110.334112:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877110.334116:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 1029 00000100:00080000:2.0:1788877110.334121:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877110.334126:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877110.334139:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511336320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877110.334236:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511335936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877110.334364:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877110.334452:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877110.334502:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511334272:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877110.334512:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511333888:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877110.334517:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334144:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877110.334521:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2398:1875771511333760:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877110.334526:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334016:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877110.334597:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511335424:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877110.334605:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2397:1875771511334784:192.168.203.111@tcp:400:kworker.0 00000100:00080000:2.0:1788877110.334676:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877110.334682:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877110.334685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877110.334688:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1788877110.334813:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336320:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877112.061558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877113.086421:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877114.429497:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877104/real 1788877104] req@ffffa06188699180 x1875771511335296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877114 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'ptlrpcd_rcv.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877114.429539:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511335296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877114 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_rcv.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877114.429556:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877114.429585:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877114.429591:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511335296:192.168.203.111@tcp:38:ptlrpcd_rcv.0 00000100:00080000:3.0:1788877114.429667:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877114.429673:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877114.429676:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877114.429679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877114.429682:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877114.429688:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2096 00000100:00080000:3.0:1788877114.429699:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788877114.429706:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877114.429720:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877114.429754:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511336448/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877114.429778:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877114.429782:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877114.429785:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877114.429788:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877114.429791:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877114.429794:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1788877114.429852:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336448:192.168.203.111@tcp:38:kworker.0 00010000:00010000:3.0:1788877115.133701:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1788877115.453902:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877099/real 1788877099] req@ffffa06188873b80 x1875771511334656/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877115 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877115.453952:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06188873b80 x1875771511334656/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877115 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877115.453965:0:2398:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffffa061893a2800 already not connected (conn 6, was 5): CONNECTING 00000100:00000400:3.0:1788877115.453973:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877099/real 1788877099] req@ffffa06188870700 x1875771511334528/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877115 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877115.453985:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06188870700 x1875771511334528/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877115 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877115.453992:0:2398:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffffa061893a3000 already not connected (conn 6, was 5): CONNECTING 00000100:00000400:3.0:1788877115.453998:0:2398:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877099/real 1788877099] req@ffffa06188873480 x1875771511334400/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877115 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877115.454016:0:2398:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa06188873480 x1875771511334400/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877115 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877115.454023:0:2398:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa061893a2000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:3.0:1788877115.454034:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334656:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877115.454040:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334528:192.168.203.111@tcp:400:kworker.0 00000100:00100000:3.0:1788877115.454044:0:2398:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:f0204769-56bd-4f5c-ab97-74c450c9e072:2398:1875771511334400:192.168.203.111@tcp:400:kworker.0 00010000:00010000:1.0:1788877116.159678:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877118.205854:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877119.230364:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877119.487552:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877119.487557:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877119.487560:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877119.487563:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877119.487565:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877119.487569:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877119.487571:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877119.487574:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877119.487577:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877119.487580:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1788877120.574017:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877110/real 1788877110] req@ffffa061888adc00 x1875771511335936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877120.574065:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511335936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877120.574083:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877110/real 1788877110] req@ffffa061888af100 x1875771511336064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877120.574095:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511336064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877120.574103:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877110/real 1788877110] req@ffffa061888afb80 x1875771511336192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877120.574114:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511336192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877120.574122:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877110/real 1788877110] req@ffffa061888ae680 x1875771511336320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877120.574131:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511336320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877120.574154:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877120.574175:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877120.574178:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511335936:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877120.574183:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877120.574189:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877120.574192:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336064:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788877120.574196:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877120.574200:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877120.574202:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336192:192.168.203.111@tcp:8:kworker.0 00000100:00080000:1.0:1788877120.574206:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877120.574209:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877120.574211:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336320:192.168.203.111@tcp:8:kworker.0 00000100:00080000:3.0:1788877120.574412:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877120.574417:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877120.574421:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877120.574427:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2102 00000100:00080000:3.0:1788877120.574437:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788877120.574454:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877120.574493:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511336576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877120.574518:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877120.574522:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877120.574525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877120.574528:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877120.574530:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877120.574533:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2102 00000100:00080000:3.0:1788877120.574538:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788877120.574549:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877120.574560:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877120.574572:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511336704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877120.574582:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788877120.574586:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511336576:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877120.574587:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877120.574590:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877120.574597:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2102 00000100:00080000:3.0:1788877120.574601:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788877120.574605:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877120.574611:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877120.574631:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511336832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877120.574640:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877120.574644:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877120.574646:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877120.574649:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2102 00000100:00080000:3.0:1788877120.574652:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1788877120.574656:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877120.574660:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877120.574671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511336960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877120.574687:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877120.574717:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877120.574736:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336960:192.168.203.111@tcp:8:kworker.0 00000100:00000400:2.0:1788877120.637555:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877104/real 1788877104] req@ffffa061888af480 x1875771511335040/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877120.637600:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888af480 x1875771511335040/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877120.637609:0:2397:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffffa061893a3000 already not connected (conn 7, was 5): CONNECTING 00000100:00000400:2.0:1788877120.637617:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877104/real 1788877104] req@ffffa061888ac000 x1875771511334912/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877120.637629:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888ac000 x1875771511334912/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877120.637636:0:2397:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa061893a2000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:2.0:1788877120.637642:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877104/real 1788877104] req@ffffa061888ae300 x1875771511335168/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877120.637652:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888ae300 x1875771511335168/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877120 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877120.637664:0:2397:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffffa061893a2800 already not connected (conn 7, was 5): CONNECTING 00000100:00100000:2.0:1788877120.637683:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335040:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877120.637689:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511334912:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877120.637694:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335168:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877121.277616:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877122.301980:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877124.349517:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877125.373792:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877125.757562:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877125.757570:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877125.757573:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877125.757577:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877125.757580:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877125.757583:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877125.757585:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877125.757588:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877125.757591:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877125.757594:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:2.0:1788877125.757718:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877109/real 1788877109] req@ffffa061888aca80 x1875771511335680/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877125 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877125.757756:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888aca80 x1875771511335680/t0(0) o400->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877125 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877125.757765:0:2397:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffffa061893a3000 already not connected (conn 7, was 5): CONNECTING 00000100:00000400:2.0:1788877125.757773:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877109/real 1788877109] req@ffffa061888ad500 x1875771511335552/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877125 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877125.757786:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888ad500 x1875771511335552/t0(0) o400->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 224/224 e 0 to 1 dl 1788877125 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877125.757793:0:2397:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffffa061893a2000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:2.0:1788877125.757799:0:2397:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877109/real 1788877109] req@ffffa061888adf80 x1875771511335808/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877125 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877125.757811:0:2397:0:(client.c:2548:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffffa061888adf80 x1875771511335808/t0(0) o400->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 224/224 e 0 to 1 dl 1788877125 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877125.757819:0:2397:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffffa061893a2800 already not connected (conn 7, was 5): CONNECTING 00000100:00100000:2.0:1788877125.757838:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335680:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877125.757845:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335552:192.168.203.111@tcp:400:kworker.0 00000100:00100000:2.0:1788877125.757850:0:2397:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:f0204769-56bd-4f5c-ab97-74c450c9e072:2397:1875771511335808:192.168.203.111@tcp:400:kworker.0 00010000:00010000:3.0:1788877127.421500:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877128.445616:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788877129.789900:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877114/real 1788877114] req@ffffa06188872680 x1875771511336448/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877129 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877129.789953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511336448/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877129 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877129.789990:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877129.790038:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877129.790041:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336448:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788877129.790156:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877129.790166:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877129.790168:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877129.790172:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877129.790176:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877129.790187:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2106 00000100:00080000:3.0:1788877129.790200:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1788877129.790208:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877129.790220:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877129.790251:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511337088/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877129.790271:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877129.790273:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877129.790275:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877129.790277:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877129.790279:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877129.790281:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1788877129.790308:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337088:192.168.203.111@tcp:38:kworker.0 00010000:00010000:3.0:1788877130.493537:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877131.517640:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877133.565512:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1788877134.205519:0:2386:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.111@tcp (192.168.203.111:988) timed out; the network or node may be down. 00010000:00010000:1.0:1788877134.590021:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877134.973625:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877134.973640:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877134.973646:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877134.973650:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877134.973656:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877134.973659:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877134.973662:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877134.973666:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877134.973672:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877134.973679:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1788877135.933504:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877120/real 1788877120] req@ffffa06188870700 x1875771511336704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877135.933558:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511336704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877135.933571:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877120/real 1788877120] req@ffffa06188873b80 x1875771511336832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877135.933585:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511336832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877135.933596:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877120/real 1788877120] req@ffffa06188872300 x1875771511336960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877135.933609:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511336960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877135.933620:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877120/real 1788877120] req@ffffa06188873480 x1875771511336576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877135.933632:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511336576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877135 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877135.933654:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877135.933685:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877135.933690:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336704:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788877135.933696:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877135.933704:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877135.933707:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336832:192.168.203.111@tcp:8:kworker.0 00000100:00080000:2.0:1788877135.933712:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877135.933717:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877135.933723:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511336960:192.168.203.111@tcp:8:kworker.0 00000100:00080000:2.0:1788877135.933727:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877135.933733:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877135.933736:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511336576:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877135.933773:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877135.933782:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877135.933786:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877135.933804:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2112 00000100:00080000:3.0:1788877135.933820:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1788877135.933829:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877135.933878:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511337216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877135.933910:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877135.933914:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1788877135.933918:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877135.933921:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877135.933923:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877135.933927:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2112 00000100:00080000:3.0:1788877135.933933:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1788877135.933938:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877135.933948:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877135.933956:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511337216:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877135.933966:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511337344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877135.933978:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877135.933994:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877135.933997:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877135.934002:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2112 00000100:00080000:3.0:1788877135.934006:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1788877135.934010:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877135.934015:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877135.934029:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511337472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877135.934038:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877135.934042:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877135.934043:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877135.934047:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2112 00000100:00080000:3.0:1788877135.934062:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1788877135.934066:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877135.934071:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877135.934083:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511337600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877135.934146:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877135.934196:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877135.934221:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337600:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877136.637496:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877137.661610:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877139.710161:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877140.733660:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877141.117547:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877141.117558:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877141.117562:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877141.117565:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877141.117568:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877141.117571:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877141.117574:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877141.117580:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1788877141.117582:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877141.117586:0:34825:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1788877142.781552:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877143.805590:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877145.853543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877146.877564:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877148.925614:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788877149.245589:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1788877129/real 1788877129] req@ffffa06188871f80 x1875771511337088/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877149 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877149.245637:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511337088/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877149 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877149.245663:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877149.245696:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877149.245700:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337088:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877149.246078:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337728:192.168.203.111@tcp:38:kworker.0 00010000:00010000:1.0:1788877149.949554:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877151.997542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877153.021973:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877154.429584:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.429594:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788877154.429599:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.429602:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788877154.429606:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.429609:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788877154.429612:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.429615:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1788877154.429617:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.429620:0:29258:0:(pinger.c:247:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000400:00000100:3.0:1788877154.429693:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.111:988 00000400:00000100:3.0:1788877154.429740:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788877154.429753:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877154.429782:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877154.429787:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877154.429793:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877154.429798:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788877154.429827:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877149/real 1788877154] req@ffffa06198afd180 x1875771511337728/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877154.429849:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511337728/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877154.429859:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877154.429878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877154.429883:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337728:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877154.429890:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877135/real 1788877154] req@ffffa06188872a00 x1875771511337344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877154.429920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511337344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877154.429927:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877154.429932:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877154.429935:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877154.429941:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877135/real 1788877154] req@ffffa06188873100 x1875771511337472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877154.429951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511337472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877154.429958:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877154.429962:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877154.429965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877154.429971:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877135/real 1788877154] req@ffffa06188871500 x1875771511337600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877154.429980:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511337600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877154.429987:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877154.429991:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877154.429994:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877154.430000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877135/real 1788877154] req@ffffa06188873800 x1875771511337216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877154.430011:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511337216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877154.430018:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877154.430021:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877154.430024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511337216:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877154.430097:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.430102:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877154.430105:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877154.430110:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2127 00000100:00080000:1.0:1788877154.430116:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1788877154.430122:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877154.430146:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511337856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877154.430166:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.430169:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877154.430170:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877154.430174:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2141 00000100:00080000:1.0:1788877154.430177:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877154.430181:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877154.430189:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877154.430199:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511337984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877154.430207:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.430210:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877154.430211:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877154.430214:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2127 00000100:00080000:1.0:1788877154.430217:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1788877154.430221:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877154.430225:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877154.430233:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511338112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877154.430240:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.430243:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877154.430244:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877154.430247:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2127 00000100:00080000:1.0:1788877154.430250:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1788877154.430253:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877154.430257:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877154.430264:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511338240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877154.430271:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877154.430274:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877154.430276:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877154.430279:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2127 00000100:00080000:1.0:1788877154.430282:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 15s 00000100:00080000:1.0:1788877154.430285:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877154.430288:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877154.430295:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511338368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877154.430348:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511337856:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877154.430428:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877154.430458:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877154/real 1788877154] req@ffffa06188699180 x1875771511337856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877154.430470:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511337856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877154.430479:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877154.430483:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877154.430486:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511337856:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877154.430598:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877154.430623:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877154.430635:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877154/real 1788877154] req@ffffa0618869b800 x1875771511337984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877154.430647:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511337984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877154.430654:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877154.430656:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877154.430659:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511337984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877154.430664:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338112:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877154.430680:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877154.430689:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877154/real 1788877154] req@ffffa0618869aa00 x1875771511338112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877154.430698:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511338112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877154.430704:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877154.430707:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877154.430710:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338112:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877154.430717:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877154.430732:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877154.430740:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877154/real 1788877154] req@ffffa0618869b100 x1875771511338240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877154.430750:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511338240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877154.430757:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877154.430760:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877154.430763:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877154.430767:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338368:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877154.430781:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877154.430789:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877154/real 1788877154] req@ffffa06188698380 x1875771511338368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877154.430798:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511338368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877154.430805:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877154.430808:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877154.430811:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338368:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877155.069515:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877156.093606:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877156.349653:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877157.373508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877158.141543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877158.397522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877159.165560:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877159.421509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877159.549556:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877159.549572:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877159.549578:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877159.549593:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2146 00000100:00080000:2.0:1788877159.549608:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877159.549618:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877159.549674:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511338496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877159.549710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877159.549714:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877159.549716:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877159.549719:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2146 00000100:00080000:2.0:1788877159.549722:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877159.549726:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877159.549739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877159.549751:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511338624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877159.549760:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877159.549764:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877159.549765:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877159.549769:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2146 00000100:00080000:2.0:1788877159.549773:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877159.549777:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877159.549783:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511338496:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788877159.549787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877159.549803:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511338752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877159.549812:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877159.549816:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877159.549818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877159.549822:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2146 00000100:00080000:2.0:1788877159.549826:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877159.549829:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877159.549835:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877159.549847:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511338880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877159.549855:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877159.549859:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877159.549860:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877159.549864:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2146 00000100:00080000:2.0:1788877159.549868:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877159.549885:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877159.549889:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877159.549903:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511339008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877159.549940:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877159.549984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338752:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877159.550029:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877159.550057:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339008:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877160.445911:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877161.213518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877161.469457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877162.239912:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877162.493506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877162.621725:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.111:988 00000400:00000100:1.0:1788877162.621769:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877162.621785:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877162.621822:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877162.621831:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877162.621838:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877162.621845:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788877162.621896:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877159/real 1788877162] req@ffffa061888aca80 x1875771511338752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877162.621931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511338752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877162.621942:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877162.621947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877162.621950:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338752:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877162.621958:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877159/real 1788877162] req@ffffa061888ae300 x1875771511338880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877162.621968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511338880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877162.621975:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877162.621978:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877162.621982:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877162.621988:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877159/real 1788877162] req@ffffa061888ac000 x1875771511339008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877162.621998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511339008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877162.622005:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877162.622008:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877162.622011:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339008:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877162.622018:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877159/real 1788877162] req@ffffa061888adf80 x1875771511338496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877162.622029:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511338496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877162.622036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877162.622039:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877162.622042:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511338496:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877162.622048:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877159/real 1788877162] req@ffffa061888ad500 x1875771511338624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877162.622057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511338624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877162.622065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877162.622068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877162.622071:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511338624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877163.517504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877164.285534:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877164.541513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877164.669525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877164.669537:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877164.669541:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877164.669552:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2151 00000100:00080000:2.0:1788877164.669562:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877164.669570:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877164.669607:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511339136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877164.669644:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877164.669652:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877164.669654:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877164.669660:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2151 00000100:00080000:2.0:1788877164.669665:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877164.669670:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877164.669681:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877164.669695:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511339264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877164.669705:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877164.669708:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877164.669709:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877164.669713:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2151 00000100:00080000:2.0:1788877164.669716:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877164.669720:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877164.669724:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877164.669733:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511339392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877164.669740:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877164.669744:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877164.669745:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877164.669748:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2151 00000100:00080000:2.0:1788877164.669751:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877164.669755:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877164.669760:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877164.669768:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511339520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877164.669776:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877164.669778:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877164.669780:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877164.669783:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2151 00000100:00080000:2.0:1788877164.669786:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877164.669789:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877164.669793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877164.669800:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511339648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877164.670057:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511339136:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877164.670158:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877164.670186:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339392:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877164.670204:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877164.670225:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339648:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877165.309892:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877165.565474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788877165.693756:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.111:988 00000400:00000100:3.0:1788877165.693798:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788877165.693824:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:1.0:1788877165.695677:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877164/real 1788877165] req@ffffa061888af480 x1875771511339136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877189 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877165.695897:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511339136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877189 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877165.695910:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877165.695915:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877165.695919:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511339136:192.168.203.111@tcp:250:kworker.0 00000800:00000100:3.0:1788877165.696179:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:1.0:1788877165.696523:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877164/real 1788877165] req@ffffa061888ad500 x1875771511339264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877189 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877165.696539:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511339264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877189 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877165.696548:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877165.696553:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877165.696556:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877165.696596:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:1.0:1788877165.696657:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877164/real 1788877165] req@ffffa061888adf80 x1875771511339392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877189 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877165.696670:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511339392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877189 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877165.696686:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877165.696690:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877165.696693:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339392:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877165.699693:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877165.700026:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:1.0:1788877165.700192:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877164/real 1788877165] req@ffffa061888ac000 x1875771511339520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877165.700212:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511339520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877165.700221:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877165.700225:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877165.700229:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877165.700236:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877164/real 1788877165] req@ffffa061888ae300 x1875771511339648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877165.700246:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511339648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877165.700252:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877165.700254:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877165.700257:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339648:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877166.589694:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877167.357550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877167.613489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877168.381575:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877168.638956:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877169.661514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877169.789701:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511339776:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877169.789846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877169.789886:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340032:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877169.789912:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877169.789937:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340288:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877170.429555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877170.685494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877171.453626:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877171.709469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877172.733513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877172.861655:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.111:988 00000400:00000100:1.0:1788877172.861695:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877172.861711:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877172.861740:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877172.861751:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877172.861761:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788877172.861770:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877169/real 1788877172] req@ffffa06198afd500 x1875771511339904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1788877172.861771:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788877172.861797:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511339904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877172.861819:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877172.861833:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877172.861838:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511339904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877172.861845:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877169/real 1788877172] req@ffffa06198afca80 x1875771511340032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877172.861857:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511340032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877172.861865:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877172.861868:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877172.861871:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340032:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877172.861877:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877169/real 1788877172] req@ffffa06198afe300 x1875771511340160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877172.861900:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511340160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877172.861907:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877172.861911:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877172.861914:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877172.861921:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877169/real 1788877172] req@ffffa06198afce00 x1875771511339776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877172.861934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afce00 x1875771511339776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877172.861942:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877172.861945:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877172.861948:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511339776:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877172.861986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877169/real 1788877172] req@ffffa06198afea00 x1875771511340288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877172.862000:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511340288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877172.862008:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877172.862010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877172.862014:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340288:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877173.501515:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877173.757443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877174.525582:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877174.781489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877174.909546:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877174.909554:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877174.909558:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877174.909568:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2161 00000100:00080000:3.0:1788877174.909578:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877174.909584:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877174.909628:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511340416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877174.909662:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877174.909666:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877174.909668:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877174.909672:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2161 00000100:00080000:3.0:1788877174.909676:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877174.909680:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877174.909690:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877174.909716:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511340544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877174.909727:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877174.909730:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877174.909731:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877174.909734:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2161 00000100:00080000:3.0:1788877174.909738:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877174.909742:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877174.909747:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877174.909759:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511340672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877174.909765:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877174.909768:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877174.909770:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877174.909773:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2161 00000100:00080000:3.0:1788877174.909777:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877174.909781:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877174.909785:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877174.909794:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511340800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877174.909819:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877174.909823:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877174.909825:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877174.909829:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2161 00000100:00080000:3.0:1788877174.909835:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877174.909839:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877174.909843:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877174.909850:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511340416:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877174.909869:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511340928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788877174.909928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877174.909967:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877174/real 1788877174] req@ffffa06188873800 x1875771511340416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877174.909987:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511340416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877174.909997:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877174.910003:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877174.910007:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511340416:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877174.910016:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877174.910035:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877174.910047:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877174/real 1788877174] req@ffffa06188871500 x1875771511340544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877174.910057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511340544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877174.910065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877174.910068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877174.910071:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877174.910077:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340672:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877174.910095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877174.910105:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877174/real 1788877174] req@ffffa06188873100 x1875771511340672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877174.910115:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511340672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877174.910123:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877174.910126:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877174.910129:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340672:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877174.910134:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877174.910151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877174.910160:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877174/real 1788877174] req@ffffa06188872a00 x1875771511340800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877174.910172:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511340800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877174.910179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877174.910182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877174.910185:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877174.910211:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340928:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877174.910227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877174.910236:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877174/real 1788877174] req@ffffa06188871f80 x1875771511340928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877174.910246:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511340928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877174.910254:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877174.910256:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877174.910259:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511340928:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877175.805462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877176.573517:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877176.829499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877177.597880:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877177.853520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877178.877523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877179.645544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877179.901539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877180.029504:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877180.029513:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877180.029517:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877180.029526:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2166 00000100:00080000:2.0:1788877180.029536:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877180.029543:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877180.029578:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511341056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877180.029609:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877180.029614:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877180.029617:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877180.029623:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2166 00000100:00080000:2.0:1788877180.029629:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877180.029635:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877180.029645:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877180.029664:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511341184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877180.029674:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877180.029677:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877180.029678:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877180.029681:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2166 00000100:00080000:2.0:1788877180.029684:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877180.029687:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877180.029691:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877180.029699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511341312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877180.029705:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877180.029708:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877180.029708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877180.029710:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2166 00000100:00080000:2.0:1788877180.029713:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877180.029718:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877180.029724:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877180.029731:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511341440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877180.029741:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877180.029743:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877180.029745:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877180.029747:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2166 00000100:00080000:2.0:1788877180.029749:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877180.029752:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877180.029756:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877180.029763:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511341568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1788877180.669588:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877180.925477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877181.949498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877182.717544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877182.973520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877183.741560:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877183.997512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877185.021469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877185.149497:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877185.149504:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877185.149508:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877185.149519:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2171 00000100:00080000:2.0:1788877185.149530:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877185.149538:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877185.149576:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511341696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877185.149600:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877185.149602:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877185.149604:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877185.149606:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2171 00000100:00080000:2.0:1788877185.149609:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877185.149612:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877185.149621:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877185.149630:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511341824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877185.149642:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877185.149645:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877185.149646:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877185.149649:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2171 00000100:00080000:2.0:1788877185.149650:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877185.149652:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877185.149655:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877185.149662:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511341952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877185.149674:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877185.149676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877185.149677:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877185.149681:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2171 00000100:00080000:2.0:1788877185.149684:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877185.149687:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877185.149692:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877185.149704:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511342080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877185.149725:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877185.149728:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877185.149732:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877185.149740:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2171 00000100:00080000:2.0:1788877185.149746:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877185.149759:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877185.149765:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877185.149779:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511342208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1788877185.789544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877186.045461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877186.813530:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877187.069484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877188.093517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788877188.221661:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.111:988 00000400:00000100:3.0:1788877188.221707:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788877188.221722:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877188.221778:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877188.221788:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877188.221797:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877188.221809:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00010000:00010000:3.0:1788877188.861580:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877189.117471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877189.885623:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877190.141547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877190.269469:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877190.269476:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877190.269479:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877190.269488:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2176 00000100:00080000:2.0:1788877190.269497:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877190.269503:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877190.269530:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511342336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877190.269550:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877190.269552:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877190.269554:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877190.269556:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2176 00000100:00080000:2.0:1788877190.269558:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877190.269567:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877190.269574:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877190.269583:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511342464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877190.269598:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877190.269600:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877190.269601:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877190.269603:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2176 00000100:00080000:2.0:1788877190.269606:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877190.269608:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877190.269610:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877190.269618:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511342592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877190.269624:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877190.269627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877190.269628:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877190.269629:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2176 00000100:00080000:2.0:1788877190.269631:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877190.269633:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877190.269636:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877190.269642:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511342720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877190.269651:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877190.269654:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877190.269656:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877190.269659:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2176 00000100:00080000:2.0:1788877190.269662:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877190.269665:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877190.269669:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877190.269679:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511342848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877191.165506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877191.933578:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877192.189514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877192.957624:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877193.213463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877194.237544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877195.005657:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877195.261517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877195.389562:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877195.389587:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877195.389592:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877195.389607:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2182 00000100:00080000:3.0:1788877195.389623:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877195.389633:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877195.389682:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511342976/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877195.389732:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877195.389737:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877195.389739:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877195.389743:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2182 00000100:00080000:3.0:1788877195.389747:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877195.389751:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877195.389763:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877195.389777:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511343104/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877195.389785:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877195.389789:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877195.389790:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877195.389794:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2182 00000100:00080000:3.0:1788877195.389797:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877195.389801:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877195.389805:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877195.389814:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511343232/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877195.389822:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877195.389826:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877195.389828:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877195.389832:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2182 00000100:00080000:3.0:1788877195.389835:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877195.389839:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877195.389843:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877195.389854:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511343360/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877195.389863:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877195.389866:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877195.389868:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877195.389872:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2182 00000100:00080000:3.0:1788877195.389875:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877195.389878:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877195.389882:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877195.389891:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511343488/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1788877196.029564:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877196.285459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877197.309515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877198.077545:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877198.333552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877199.101597:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877199.357463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877200.381463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877200.509722:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511343616:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877200.509790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877200.509827:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877200/real 1788877200] req@ffffa06198afc000 x1875771511343616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877200.509847:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511343616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877200.509859:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877200.509864:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877200.509869:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511343616:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877200.509879:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511343744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877200.509900:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877200.509912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877200/real 1788877200] req@ffffa06198afc700 x1875771511343744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877200.509923:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511343744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877200.509943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877200.509947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877200.509951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511343744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877200.509957:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511343872:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877200.509976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877200.509986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877200/real 1788877200] req@ffffa06198afdc00 x1875771511343872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877200.509997:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511343872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877200.510009:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877200.510013:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877200.510017:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511343872:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877200.510042:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877200.510059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877200.510069:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877200/real 1788877200] req@ffffa06198aff480 x1875771511344000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877200.510079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511344000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877200.510086:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877200.510090:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877200.510094:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877200.510100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344128:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877200.510117:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877200.510130:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877200/real 1788877200] req@ffffa06198affb80 x1875771511344128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877200.510145:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511344128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877200.510152:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877200.510156:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877200.510158:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344128:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877201.149552:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877201.405554:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877202.173628:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877202.429565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877203.453494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877204.221513:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877204.477501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877205.245555:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877205.501498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877205.629526:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877205.629539:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877205.629544:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877205.629557:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2192 00000100:00080000:3.0:1788877205.629571:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877205.629582:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877205.629629:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511344256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877205.629662:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877205.629666:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877205.629667:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877205.629671:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2192 00000100:00080000:3.0:1788877205.629674:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877205.629678:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877205.629688:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877205.629699:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511344384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877205.629708:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877205.629711:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511344256:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877205.629711:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877205.629714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877205.629720:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2192 00000100:00080000:3.0:1788877205.629723:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877205.629725:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877205.629729:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877205.629740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511344512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877205.629750:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877205.629754:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877205.629756:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877205.629760:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2192 00000100:00080000:3.0:1788877205.629774:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877205.629778:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877205.629783:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877205.629792:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511344640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877205.629798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877205.629801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877205.629802:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877205.629805:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2192 00000100:00080000:3.0:1788877205.629808:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877205.629811:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877205.629814:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877205.629821:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511344768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877205.629829:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877205.629865:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344512:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877205.629885:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877205.629908:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344768:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877206.525525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877207.293485:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877207.549507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877208.317489:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877208.573463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877208.701634:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877208.701684:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877208.701698:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877208.701734:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877208.701742:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877208.701749:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877208.701758:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788877208.701784:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877205/real 1788877208] req@ffffa06188872d80 x1875771511344768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877208.701806:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511344768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877208.701815:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877208.701819:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877208.701822:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344768:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877208.701827:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877205/real 1788877208] req@ffffa06188871c00 x1875771511344384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877208.701835:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511344384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877208.701840:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877208.701842:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877208.701844:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877208.701847:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877205/real 1788877208] req@ffffa06188871880 x1875771511344512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877208.701853:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511344512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877208.701857:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877208.701859:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877208.701861:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344512:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877208.701864:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877205/real 1788877208] req@ffffa06188870a80 x1875771511344640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877208.701870:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511344640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877208.701874:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877208.701876:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877208.701878:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511344640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877208.701882:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877205/real 1788877208] req@ffffa06188870380 x1875771511344256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877208.701893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511344256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877208.701900:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877208.701903:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877208.701907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511344256:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877209.597518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877210.365532:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877210.621472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877210.749492:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877210.749501:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877210.749505:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877210.749513:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2197 00000100:00080000:3.0:1788877210.749523:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877210.749529:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877210.749557:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511344896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877210.749584:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877210.749587:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877210.749589:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877210.749593:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2197 00000100:00080000:3.0:1788877210.749596:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877210.749600:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877210.749610:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877210.749623:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511345024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877210.749643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511344896:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877210.749648:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877210.749653:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877210.749655:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877210.749659:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2197 00000100:00080000:3.0:1788877210.749663:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877210.749667:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877210.749675:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877210.749689:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511345152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877210.749699:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877210.749702:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1788877210.749706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877210.749707:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877210.749714:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2197 00000100:00080000:3.0:1788877210.749720:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877210.749724:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877210.749730:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877210.749744:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877210/real 1788877210] req@ffffa06188870380 x1875771511344896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877210.749745:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511345280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877210.749766:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877210.749773:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877210.749777:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877210.749782:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2197 00000100:00080000:3.0:1788877210.749787:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877210.749789:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511344896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877210.749794:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877210.749803:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1788877210.749806:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877210.749812:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877210.749818:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511345408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877210.749819:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511344896:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877210.749847:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877210.749869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877210.749879:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877210/real 1788877210] req@ffffa06188870a80 x1875771511345024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877210.749890:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511345024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877210.749897:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877210.749900:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877210.749903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877210.749908:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345152:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877210.749923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877210.749948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877210/real 1788877210] req@ffffa06188871880 x1875771511345152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877210.749958:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511345152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877210.749964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877210.749967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877210.749970:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345152:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877210.749975:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877210.749992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877210.750000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877210/real 1788877210] req@ffffa06188871c00 x1875771511345280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877210.750012:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511345280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877210.750019:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877210.750021:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877210.750024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877210.750030:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345408:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877210.750044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877210.750053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877210/real 1788877210] req@ffffa06188872d80 x1875771511345408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877210.750062:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511345408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877210.750068:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877210.750072:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877210.750074:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345408:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877211.389561:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877211.645518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877212.669451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877213.437543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877213.693504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877214.461518:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877214.717443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877215.741499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877215.869462:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877215.869472:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877215.869476:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877215.869486:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2202 00000100:00080000:3.0:1788877215.869496:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877215.869505:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877215.869538:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511345536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877215.869572:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877215.869577:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877215.869579:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877215.869583:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2202 00000100:00080000:3.0:1788877215.869586:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877215.869591:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877215.869600:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877215.869624:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511345664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877215.869632:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511345536:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877215.869643:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877215.869647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877215.869649:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877215.869653:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2202 00000100:00080000:3.0:1788877215.869657:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877215.869661:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877215.869666:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877215.869677:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511345792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788877215.869685:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877215.869692:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877215.869696:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877215.869698:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877215.869703:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2202 00000100:00080000:3.0:1788877215.869707:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877215.869711:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877215.869717:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877215.869723:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877215/real 1788877215] req@ffffa06188872680 x1875771511345536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877215.869729:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511345920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877215.869746:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877215.869749:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788877215.869751:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511345536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877215.869752:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877215.869760:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2202 00000100:00080000:3.0:1788877215.869766:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877215.869773:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:1.0:1788877215.869775:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788877215.869778:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1788877215.869780:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877215.869785:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511345536:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877215.869792:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511346048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877215.869806:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877215.869824:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877215.869834:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877215/real 1788877215] req@ffffa06188870700 x1875771511345664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877215.869842:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511345664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877215.869847:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877215.869850:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877215.869853:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877215.869858:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345792:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877215.869876:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877215.869886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877215/real 1788877215] req@ffffa06188873b80 x1875771511345792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877215.869897:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511345792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877215.869903:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877215.869905:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877215.869907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345792:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877215.869911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877215.869925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877215.869935:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877215/real 1788877215] req@ffffa06188872300 x1875771511345920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877215.869946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511345920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877215.869953:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877215.869955:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877215.869959:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511345920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877215.869964:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346048:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877215.869982:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877215.870005:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877215/real 1788877215] req@ffffa06188873480 x1875771511346048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877215.870016:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511346048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877215.870024:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877215.870027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877215.870030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346048:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877216.509469:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877216.765514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877217.533591:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877217.789487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877218.813516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877219.581526:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877219.837500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877220.605570:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877220.861492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877220.989513:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877220.989525:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877220.989529:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877220.989539:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2207 00000100:00080000:3.0:1788877220.989552:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877220.989559:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877220.989608:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511346176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877220.989633:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877220.989637:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877220.989640:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877220.989645:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2207 00000100:00080000:3.0:1788877220.989649:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877220.989653:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877220.989662:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877220.989681:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511346176:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877220.989682:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511346304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877220.989696:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877220.989698:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877220.989700:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877220.989704:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2207 00000100:00080000:3.0:1788877220.989707:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877220.989711:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877220.989716:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877220.989726:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511346432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877220.989735:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877220.989739:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877220.989740:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877220.989743:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2207 00000100:00080000:3.0:1788877220.989747:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788877220.989750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877220.989754:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877220.989761:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877220.989772:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511346560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877220.989780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877220.989783:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877220.989785:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788877220.989788:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877220/real 1788877220] req@ffffa06188871f80 x1875771511346176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877220.989791:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2207 00000100:00080000:3.0:1788877220.989797:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877220.989804:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877220.989809:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877220.989814:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511346176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877220.989821:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511346688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877220.989828:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877220.989835:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877220.989840:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511346176:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877220.989863:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877220.989881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877220.989891:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877220/real 1788877220] req@ffffa06188872a00 x1875771511346304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877220.989903:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511346304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877220.989910:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877220.989913:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877220.989916:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877220.989921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346432:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877220.989939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877220.989948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877220/real 1788877220] req@ffffa06188873100 x1875771511346432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877220.989958:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511346432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877220.989966:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877220.989969:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877220.989973:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346432:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877220.989979:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877220.990001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877220.990011:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877220/real 1788877220] req@ffffa06188871500 x1875771511346560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877220.990021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511346560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877220.990028:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877220.990031:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877220.990034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877220.990040:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346688:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877220.990056:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877220.990065:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877220/real 1788877220] req@ffffa06188873800 x1875771511346688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877220.990082:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511346688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877220.990089:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877220.990092:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877220.990095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346688:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877221.885480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877222.653602:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877222.909503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877223.677526:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877223.933483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877224.957487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877225.725483:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877225.981504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877226.109489:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877226.109498:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877226.109503:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877226.109514:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2212 00000100:00080000:3.0:1788877226.109525:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877226.109533:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877226.109582:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511346816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877226.109610:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877226.109614:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877226.109616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877226.109621:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2212 00000100:00080000:3.0:1788877226.109625:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877226.109629:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877226.109639:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877226.109653:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511346944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877226.109665:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511346816:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877226.109666:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877226.109678:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877226.109680:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877226.109685:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2212 00000100:00080000:3.0:1788877226.109689:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877226.109694:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877226.109698:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877226.109708:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511347072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877226.109715:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877226.109718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788877226.109720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877226.109721:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877226.109727:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2212 00000100:00080000:3.0:1788877226.109730:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877226.109733:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877226.109737:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877226.109745:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511347200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877226.109751:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788877226.109753:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877226/real 1788877226] req@ffffa06188873800 x1875771511346816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877226.109754:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877226.109757:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877226.109764:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2212 00000100:00080000:3.0:1788877226.109774:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788877226.109775:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511346816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877226.109780:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877226.109786:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1788877226.109791:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788877226.109795:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511347328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877226.109796:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877226.109804:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511346816:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877226.109827:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877226.109842:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877226.109853:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877226/real 1788877226] req@ffffa06188871500 x1875771511346944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877226.109860:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511346944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877226.109865:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877226.109867:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877226.109869:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511346944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877226.109872:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347072:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877226.109881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877226.109886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877226/real 1788877226] req@ffffa06188873100 x1875771511347072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877226.109896:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511347072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877226.109903:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877226.109906:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877226.109909:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347072:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877226.109914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877226.109933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877226.109952:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877226/real 1788877226] req@ffffa06188872a00 x1875771511347200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877226.109964:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511347200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877226.109972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877226.109975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877226.109978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877226.109983:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347328:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877226.109997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877226.110004:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877226/real 1788877226] req@ffffa06188871f80 x1875771511347328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877226.110010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511347328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877226.110015:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877226.110017:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877226.110019:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347328:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877226.749588:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877227.005488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877228.029474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877228.797592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877229.053536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877229.821544:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877230.077469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877231.101511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877231.229473:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877231.229490:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877231.229494:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877231.229504:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2217 00000100:00080000:3.0:1788877231.229515:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877231.229528:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877231.229575:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511347456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877231.229601:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877231.229605:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877231.229607:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877231.229611:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2217 00000100:00080000:3.0:1788877231.229615:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877231.229621:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877231.229630:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877231.229643:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511347584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877231.229652:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877231.229656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1788877231.229659:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511347456:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877231.229659:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877231.229669:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2217 00000100:00080000:3.0:1788877231.229672:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877231.229676:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877231.229680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877231.229692:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511347712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877231.229700:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877231.229703:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877231.229706:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877231.229709:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2217 00000100:00080000:3.0:1788877231.229713:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788877231.229715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877231.229721:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877231.229726:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877231.229737:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511347840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877231.229744:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877231/real 1788877231] req@ffffa06188873480 x1875771511347456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877231.229748:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877231.229756:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877231.229760:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877231.229769:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2217 00000100:00080000:3.0:1788877231.229773:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788877231.229785:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511347456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877231.229786:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877231.229794:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1788877231.229797:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877231.229802:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877231.229809:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511347456:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877231.229810:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511347968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877231.229829:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877231.229845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877231.229853:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877231/real 1788877231] req@ffffa06188872300 x1875771511347584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877231.229860:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511347584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877231.229864:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877231.229866:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877231.229868:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877231.229871:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347712:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877231.229880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877231.229885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877231/real 1788877231] req@ffffa06188873b80 x1875771511347712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877231.229892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511347712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877231.229899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877231.229901:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877231.229903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347712:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877231.229906:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877231.229915:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877231.229926:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877231/real 1788877231] req@ffffa06188870700 x1875771511347840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877231.229932:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511347840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877231.229936:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877231.229938:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877231.229941:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877231.229947:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347968:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877231.229963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877231.229974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877231/real 1788877231] req@ffffa06188872680 x1875771511347968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877231.229985:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511347968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877231.229991:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877231.229993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877231.229995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511347968:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877231.869496:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877232.125460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877232.893546:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877233.149484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877234.173483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877234.941469:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877235.197461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877235.965566:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877236.221453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877236.349715:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511348096:192.168.203.111@tcp:250:kworker.0 00000800:00000100:1.0:1788877236.349793:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877236.349830:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877236/real 1788877236] req@ffffa06198afed80 x1875771511348096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877236.349859:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511348096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877236.349882:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877236.349887:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877236.349891:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511348096:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877236.349914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877236.349943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877236.349961:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877236/real 1788877236] req@ffffa06198aff800 x1875771511348224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877236.349977:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511348224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877236.349988:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877236.349993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877236.349996:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877236.350003:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348352:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877236.350024:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877236.350035:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877236/real 1788877236] req@ffffa06198afdf80 x1875771511348352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877236.350046:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511348352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877236.350053:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877236.350057:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877236.350060:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348352:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877236.350072:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877236.350094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877236.350107:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877236/real 1788877236] req@ffffa06198afd880 x1875771511348480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877236.350117:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511348480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877236.350124:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877236.350127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877236.350131:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877236.350138:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348608:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877236.350158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877236.350168:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877236/real 1788877236] req@ffffa06198affb80 x1875771511348608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877236.350180:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511348608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877236.350187:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877236.350191:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877236.350194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348608:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877237.245544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877238.013629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877238.269516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877239.037549:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877239.293501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877240.317551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877241.085510:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877241.341476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877241.469532:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877241.469544:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877241.469550:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877241.469563:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2228 00000100:00080000:3.0:1788877241.469578:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877241.469587:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877241.469636:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511348736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877241.469658:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877241.469661:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877241.469663:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877241.469666:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2228 00000100:00080000:3.0:1788877241.469669:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877241.469673:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877241.469682:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877241.469695:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511348864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877241.469706:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877241.469710:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877241.469713:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788877241.469720:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511348736:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877241.469722:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2228 00000100:00080000:3.0:1788877241.469730:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877241.469738:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877241.469744:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877241.469759:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511348992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877241.469769:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877241.469772:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877241.469774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877241.469777:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2228 00000100:00080000:3.0:1788877241.469780:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877241.469784:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877241.469789:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877241.469799:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511349120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877241.469804:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877241.469807:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877241.469809:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877241.469811:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2228 00000100:00080000:3.0:1788877241.469814:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877241.469816:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877241.469820:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877241.469828:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511349248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877241.469851:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877241.469892:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348992:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877241.469911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877241.469935:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349248:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877242.109583:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877242.365502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877243.389482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877244.157550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877244.413460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788877244.541680:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788877244.541724:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788877244.541738:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877244.541772:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877244.541782:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877244.541789:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877244.541797:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:1.0:1788877244.541824:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877241/real 1788877244] req@ffffa06188871c00 x1875771511348864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877244.541845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511348864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877244.541857:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877244.541861:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877244.541865:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877244.541873:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877241/real 1788877244] req@ffffa06188871880 x1875771511348992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877244.541885:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511348992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877244.541892:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877244.541895:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877244.541898:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511348992:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877244.541904:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877241/real 1788877244] req@ffffa06188870a80 x1875771511349120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877244.541915:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511349120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877244.541923:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877244.541926:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877244.541930:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877244.541939:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877241/real 1788877244] req@ffffa06188870380 x1875771511349248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877244.541951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511349248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877244.541958:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877244.541961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877244.541964:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349248:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877244.541971:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877241/real 1788877244] req@ffffa06188872d80 x1875771511348736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877244.541982:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511348736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877244.541988:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877244.541991:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877244.541994:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511348736:192.168.203.111@tcp:250:kworker.0 00010000:00010000:1.0:1788877245.181541:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877245.437449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877246.461457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877246.589502:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877246.589512:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877246.589516:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877246.589524:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2233 00000100:00080000:1.0:1788877246.589536:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877246.589543:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877246.589576:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511349376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877246.589608:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877246.589611:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877246.589613:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877246.589617:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2233 00000100:00080000:1.0:1788877246.589620:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877246.589624:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877246.589632:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877246.589645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511349504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877246.589655:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877246.589660:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877246.589662:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877246.589662:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511349376:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877246.589668:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2233 00000100:00080000:1.0:1788877246.589672:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877246.589675:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877246.589683:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877246.589696:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511349632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877246.589704:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877246.589707:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877246.589708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877246.589712:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2233 00000100:00080000:1.0:1788877246.589716:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877246.589720:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877246.589725:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877246.589735:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511349760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877246.589743:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1788877246.589748:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877246.589750:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877246.589753:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877246.589758:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2233 00000100:00080000:1.0:1788877246.589762:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877246.589767:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877246.589772:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877246.589783:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511349888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877246.589787:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877246/real 1788877246] req@ffffa06188699c00 x1875771511349376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877246.589820:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511349376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877246.589830:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877246.589834:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877246.589838:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511349376:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877246.589860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877246.589880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877246.589892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877246/real 1788877246] req@ffffa06188698380 x1875771511349504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877246.589910:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511349504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877246.589917:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877246.589920:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877246.589923:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877246.589928:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349632:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877246.589946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877246.589955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877246/real 1788877246] req@ffffa0618869b100 x1875771511349632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877246.589966:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511349632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877246.589972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877246.589975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877246.589978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349632:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877246.589984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877246.590000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877246.590009:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877246/real 1788877246] req@ffffa0618869aa00 x1875771511349760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877246.590019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511349760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877246.590031:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877246.590034:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877246.590037:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877246.590042:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349888:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877246.590057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877246.590066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877246/real 1788877246] req@ffffa0618869b800 x1875771511349888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877246.590077:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511349888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877246.590083:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877246.590086:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877246.590088:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511349888:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877247.229477:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877247.485450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877248.253547:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877248.509460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877249.533461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877250.301527:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877250.557528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877251.325574:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877251.581475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877251.709524:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877251.709534:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877251.709539:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877251.709550:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2238 00000100:00080000:1.0:1788877251.709561:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877251.709569:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877251.709606:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511350016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877251.709631:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877251.709635:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877251.709636:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877251.709641:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2238 00000100:00080000:1.0:1788877251.709645:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877251.709649:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877251.709659:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877251.709671:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511350144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877251.709682:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877251.709685:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877251.709687:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877251.709691:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2238 00000100:00080000:1.0:1788877251.709695:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877251.709699:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877251.709704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877251.709715:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511350016:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877251.709722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511350272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877251.709743:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877251.709747:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877251.709749:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877251.709754:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2238 00000100:00080000:1.0:1788877251.709758:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877251.709763:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877251.709768:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877251.709782:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511350400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877251.709792:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877251.709796:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877251.709798:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877251.709803:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2238 00000100:00080000:1.0:1788877251.709807:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877251.709813:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788877251.709821:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788877251.709824:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877251.709840:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511350528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877251.709878:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877251/real 1788877251] req@ffffa06188699180 x1875771511350016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877251.709901:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511350016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877251.709912:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877251.709917:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877251.709921:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511350016:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877251.709931:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877251.709953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877251.709965:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877251/real 1788877251] req@ffffa06188699f80 x1875771511350144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877251.709976:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511350144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877251.709983:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877251.709987:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877251.709990:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877251.710018:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350272:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877251.710034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877251.710043:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877251/real 1788877251] req@ffffa06188698000 x1875771511350272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877251.710054:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511350272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877251.710061:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877251.710065:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877251.710068:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350272:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877251.710074:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877251.710092:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877251.710102:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877251/real 1788877251] req@ffffa0618869ad80 x1875771511350400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877251.710113:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511350400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877251.710121:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877251.710124:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877251.710127:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877251.710132:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350528:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877251.710149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877251.710158:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877251/real 1788877251] req@ffffa0618869bb80 x1875771511350528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877251.710168:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511350528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877251.710175:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877251.710179:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877251.710181:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350528:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877252.605540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877253.373527:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877253.629524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877254.397581:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877254.653501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877255.677489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877256.445543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877256.701544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877256.829520:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877256.829531:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877256.829535:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877256.829547:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2243 00000100:00080000:1.0:1788877256.829560:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877256.829567:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877256.829611:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511350656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877256.829634:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877256.829636:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877256.829638:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877256.829641:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2243 00000100:00080000:1.0:1788877256.829643:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877256.829646:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877256.829655:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877256.829672:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511350784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877256.829685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877256.829690:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1788877256.829690:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511350656:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877256.829694:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877256.829705:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2243 00000100:00080000:1.0:1788877256.829710:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877256.829713:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877256.829718:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877256.829729:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511350912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877256.829736:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877256.829738:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877256.829740:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877256.829744:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2243 00000100:00080000:1.0:1788877256.829746:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877256.829748:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877256.829752:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877256.829759:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511351040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877256.829767:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877256.829770:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877256.829771:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877256.829773:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2243 00000100:00080000:1.0:1788877256.829775:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877256.829778:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877256.829781:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877256.829788:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877256.829794:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511351168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877256.829842:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877256/real 1788877256] req@ffffa0618869a680 x1875771511350656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877256.829861:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511350656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877256.829880:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877256.829884:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877256.829888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511350656:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877256.829923:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877256.829948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877256.829961:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877256/real 1788877256] req@ffffa0618869bb80 x1875771511350784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877256.829973:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511350784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877256.829981:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877256.829985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877256.829989:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877256.829995:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350912:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877256.830014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877256.830024:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877256/real 1788877256] req@ffffa0618869ad80 x1875771511350912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877256.830035:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511350912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877256.830043:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877256.830046:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877256.830049:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511350912:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877256.830056:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877256.830074:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877256.830085:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877256/real 1788877256] req@ffffa06188698000 x1875771511351040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877256.830097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511351040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877256.830104:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877256.830108:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877256.830111:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877256.830118:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351168:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877256.830135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877256.830145:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877256/real 1788877256] req@ffffa06188699f80 x1875771511351168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877256.830155:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511351168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877256.830163:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877256.830167:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877256.830170:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351168:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877257.469523:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877257.725511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877258.749499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877259.517550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877259.773514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877260.541528:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877260.797515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877261.821501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877261.949500:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877261.949511:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877261.949515:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877261.949527:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2248 00000100:00080000:3.0:1788877261.949538:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877261.949557:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877261.949596:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511351296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877261.949622:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877261.949625:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877261.949627:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877261.949631:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2248 00000100:00080000:3.0:1788877261.949635:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877261.949639:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877261.949649:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877261.949665:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511351424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877261.949675:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511351296:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877261.949682:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877261.949687:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877261.949690:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877261.949695:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2248 00000100:00080000:3.0:1788877261.949699:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877261.949706:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877261.949712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877261.949729:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511351552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877261.949739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877261.949742:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877261.949745:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877261.949748:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2248 00000100:00080000:3.0:1788877261.949754:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877261.949758:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877261.949764:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877261.949771:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877261.949780:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511351680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877261.949794:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877261.949799:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877261.949801:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877261.949806:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2248 00000100:00080000:3.0:1788877261.949812:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877261.949819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877261.949826:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877261.949829:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877261/real 1788877261] req@ffffa06188872d80 x1875771511351296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877261.949843:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511351808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877261.949865:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511351296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877261.949876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877261.949880:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877261.949885:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511351296:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877261.949910:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877261.949929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877261.949937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877261/real 1788877261] req@ffffa06188870380 x1875771511351424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877261.949946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511351424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877261.949954:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877261.949957:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877261.949960:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877261.949967:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351552:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877261.949987:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877261.949998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877261/real 1788877261] req@ffffa06188870a80 x1875771511351552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877261.950005:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511351552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877261.950014:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877261.950017:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877261.950019:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351552:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877261.950023:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877261.950036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877261.950042:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877261/real 1788877261] req@ffffa06188871880 x1875771511351680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877261.950049:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511351680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877261.950061:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877261.950063:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877261.950065:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877261.950069:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351808:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877261.950080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877261.950087:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877261/real 1788877261] req@ffffa06188871c00 x1875771511351808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877261.950094:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511351808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877261.950098:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877261.950099:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877261.950102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511351808:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877262.589518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877262.845500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877263.613561:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877263.869502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877264.893480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877265.661534:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877265.917488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877266.685529:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877266.941446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877267.069494:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877267.069508:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877267.069513:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877267.069522:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2253 00000100:00080000:3.0:1788877267.069532:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877267.069539:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877267.069572:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511351936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877267.069591:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877267.069594:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877267.069595:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877267.069598:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2253 00000100:00080000:3.0:1788877267.069600:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877267.069603:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877267.069610:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877267.069618:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511352064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877267.069625:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877267.069628:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877267.069629:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877267.069632:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2253 00000100:00080000:3.0:1788877267.069633:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877267.069636:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877267.069639:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877267.069645:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511352192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877267.069653:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511351936:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877267.069654:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877267.069659:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877267.069662:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877267.069667:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2253 00000100:00080000:3.0:1788877267.069669:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877267.069671:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877267.069674:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877267.069682:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511352320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877267.069688:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877267.069690:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877267.069692:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877267.069694:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2253 00000100:00080000:3.0:1788877267.069705:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877267.069708:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877267.069711:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877267.069718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877267.069721:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511352448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877267.069756:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877267/real 1788877267] req@ffffa06188872680 x1875771511351936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877267.069777:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511351936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877267.069788:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877267.069793:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877267.069798:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511351936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877267.069806:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877267.069827:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877267.069837:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877267/real 1788877267] req@ffffa06188870700 x1875771511352064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877267.069845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511352064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877267.069851:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877267.069853:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877267.069855:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877267.069875:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352192:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877267.069885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877267.069893:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877267/real 1788877267] req@ffffa06188873b80 x1875771511352192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877267.069900:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511352192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877267.069905:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877267.069907:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877267.069909:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352192:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877267.069912:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877267.069922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877267.069930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877267/real 1788877267] req@ffffa06188872300 x1875771511352320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877267.069942:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511352320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877267.069950:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877267.069953:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877267.069956:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877267.069963:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352448:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877267.069983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877267.069993:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877267/real 1788877267] req@ffffa06188873480 x1875771511352448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877267.070001:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511352448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877267.070007:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877267.070009:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877267.070011:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352448:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877267.965509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877268.733488:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877268.989470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877269.757562:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877270.013516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877271.037470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877271.805543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877272.061528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877272.189456:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877272.189463:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877272.189467:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877272.189476:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2258 00000100:00080000:3.0:1788877272.189486:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877272.189493:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877272.189524:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511352576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877272.189547:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877272.189550:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877272.189552:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877272.189555:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2258 00000100:00080000:3.0:1788877272.189557:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877272.189559:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877272.189566:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877272.189575:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511352704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877272.189583:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877272.189586:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877272.189587:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877272.189590:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2258 00000100:00080000:3.0:1788877272.189591:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877272.189593:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877272.189597:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877272.189599:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511352576:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877272.189606:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511352832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877272.189614:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877272.189617:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877272.189620:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877272.189624:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2258 00000100:00080000:3.0:1788877272.189630:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877272.189634:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877272.189639:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877272.189653:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511352960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877272.189656:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877272.189668:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877272.189672:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877272.189674:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877272.189678:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2258 00000100:00080000:3.0:1788877272.189682:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877272.189685:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877272.189689:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877272.189698:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877272/real 1788877272] req@ffffa06188871f80 x1875771511352576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877272.189700:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511353088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877272.189735:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511352576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877272.189747:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877272.189752:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877272.189756:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511352576:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877272.189767:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877272.189795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877272.189807:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877272/real 1788877272] req@ffffa06188872a00 x1875771511352704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877272.189819:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511352704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877272.189825:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877272.189828:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877272.189830:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877272.189850:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352832:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877272.189872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877272.189885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877272/real 1788877272] req@ffffa06188873100 x1875771511352832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877272.189896:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511352832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877272.189903:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877272.189907:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877272.189911:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352832:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877272.189918:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877272.189942:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877272.189957:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877272/real 1788877272] req@ffffa06188871500 x1875771511352960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877272.189968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511352960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877272.189979:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877272.189984:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877272.189988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511352960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877272.189996:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353088:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877272.190021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877272.190032:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877272/real 1788877272] req@ffffa06188873800 x1875771511353088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877272.190043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511353088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877272.190049:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877272.190051:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877272.190055:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353088:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877272.829570:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877273.085430:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877274.109469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877274.877554:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877275.133501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877275.901562:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877276.157501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877277.181449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877277.309547:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877277.309557:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877277.309562:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877277.309572:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2264 00000100:00080000:2.0:1788877277.309583:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877277.309591:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877277.309630:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511353216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877277.309659:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877277.309663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877277.309665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877277.309670:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2264 00000100:00080000:2.0:1788877277.309674:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877277.309680:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877277.309690:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877277.309699:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511353216:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877277.309707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511353344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877277.309726:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877277.309730:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877277.309733:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877277.309737:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2264 00000100:00080000:2.0:1788877277.309742:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877277.309746:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:1.0:1788877277.309751:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788877277.309752:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877277.309767:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511353472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877277.309779:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788877277.309783:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877277/real 1788877277] req@ffffa061888ac380 x1875771511353216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877277.309786:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877277.309790:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877277.309799:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2264 00000100:00080000:2.0:1788877277.309804:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788877277.309807:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511353216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877277.309812:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877277.309820:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1788877277.309835:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788877277.309835:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511353600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877277.309847:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788877277.309854:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788877277.309856:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511353216:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877277.309860:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877277.309865:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877277.309870:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2264 00000100:00080000:2.0:1788877277.309875:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877277.309879:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877277.309884:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877277.309889:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877277.309897:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511353728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788877277.309914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877277.309928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877277/real 1788877277] req@ffffa061888ad880 x1875771511353344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877277.309941:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511353344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877277.309950:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877277.309954:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877277.309958:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877277.309964:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353472:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877277.309985:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877277.309994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877277/real 1788877277] req@ffffa061888adc00 x1875771511353472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877277.310006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511353472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877277.310015:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877277.310018:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877277.310022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353472:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877277.310029:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877277.310049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877277.310060:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877277/real 1788877277] req@ffffa061888af100 x1875771511353600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877277.310072:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511353600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877277.310078:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877277.310080:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877277.310082:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877277.310094:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353728:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877277.310107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877277.310113:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877277/real 1788877277] req@ffffa061888afb80 x1875771511353728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877277.310120:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511353728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877277.310126:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877277.310128:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877277.310129:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353728:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877277.949566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877278.205471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877278.973543:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877279.229545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877280.253471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877281.021564:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877281.277504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877282.045520:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877282.301481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877282.429501:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877282.429510:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877282.429515:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877282.429524:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2269 00000100:00080000:2.0:1788877282.429534:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877282.429541:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877282.429574:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511353856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877282.429595:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877282.429598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877282.429600:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877282.429603:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2269 00000100:00080000:2.0:1788877282.429605:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877282.429608:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877282.429615:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877282.429626:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511353984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877282.429633:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877282.429638:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877282.429640:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877282.429643:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2269 00000100:00080000:2.0:1788877282.429646:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877282.429651:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877282.429654:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511353856:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788877282.429656:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877282.429669:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511354112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877282.429677:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877282.429679:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877282.429681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877282.429684:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2269 00000100:00080000:2.0:1788877282.429687:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877282.429690:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877282.429693:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877282.429701:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511354240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877282.429707:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877282.429709:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877282.429710:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877282.429713:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2269 00000100:00080000:2.0:1788877282.429715:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877282.429718:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:1.0:1788877282.429724:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788877282.429724:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877282.429734:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511354368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877282.429763:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877282/real 1788877282] req@ffffa061888ae680 x1875771511353856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877282.429783:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511353856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877282.429794:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877282.429799:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877282.429803:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511353856:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877282.429811:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877282.429832:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877282.429844:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877282/real 1788877282] req@ffffa061888aca80 x1875771511353984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877282.429868:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511353984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877282.429878:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877282.429882:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877282.429886:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511353984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877282.429910:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354112:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877282.429932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877282.429943:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877282/real 1788877282] req@ffffa061888ae300 x1875771511354112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877282.429957:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511354112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877282.429965:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877282.429969:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877282.429972:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354112:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877282.429979:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877282.429997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877282.430008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877282/real 1788877282] req@ffffa061888ac000 x1875771511354240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877282.430020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511354240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877282.430029:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877282.430032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877282.430036:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877282.430042:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354368:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877282.430060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877282.430071:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877282/real 1788877282] req@ffffa061888adf80 x1875771511354368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877282.430092:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511354368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877282.430101:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877282.430104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877282.430108:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354368:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877283.325485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877284.093516:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877284.349450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877285.117656:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877285.373508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877286.397464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877287.165452:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877287.421489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877287.549493:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877287.549503:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877287.549508:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877287.549518:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2274 00000100:00080000:3.0:1788877287.549528:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877287.549536:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877287.549571:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511354496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877287.549597:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877287.549601:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877287.549603:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877287.549607:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2274 00000100:00080000:3.0:1788877287.549611:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877287.549615:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877287.549624:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877287.549638:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511354624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877287.549645:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511354496:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877287.549653:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877287.549658:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877287.549660:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877287.549665:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2274 00000100:00080000:3.0:1788877287.549669:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877287.549673:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877287.549679:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877287.549691:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511354752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877287.549701:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877287.549705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877287.549709:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788877287.549710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877287.549718:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2274 00000100:00080000:3.0:1788877287.549723:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877287.549727:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877287.549732:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877287.549744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511354880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877287.549751:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877287/real 1788877287] req@ffffa06188873800 x1875771511354496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877287.549760:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877287.549766:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877287.549768:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788877287.549774:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511354496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877287.549775:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2274 00000100:00080000:3.0:1788877287.549783:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877287.549787:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877287.549790:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:1.0:1788877287.549794:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788877287.549798:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877287.549799:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511354496:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877287.549815:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511355008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877287.549825:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877287.549856:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877287.549866:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877287/real 1788877287] req@ffffa06188871500 x1875771511354624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877287.549875:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511354624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877287.549880:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877287.549882:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877287.549885:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877287.549888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354752:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877287.549899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877287.549905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877287/real 1788877287] req@ffffa06188873100 x1875771511354752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877287.549912:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511354752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877287.549917:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877287.549919:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877287.549920:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354752:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877287.549924:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877287.549933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877287.549939:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877287/real 1788877287] req@ffffa06188872a00 x1875771511354880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877287.549946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511354880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877287.549951:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877287.549953:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877287.549955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511354880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877287.549965:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355008:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877287.549974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877287.549979:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877287/real 1788877287] req@ffffa06188871f80 x1875771511355008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877287.549986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511355008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877287.549992:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877287.549993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877287.549995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355008:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877288.189574:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877288.445461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877289.469469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877290.237508:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877290.493446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877291.261528:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877291.517492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877292.541461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877292.669500:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877292.669509:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877292.669513:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877292.669523:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2279 00000100:00080000:3.0:1788877292.669533:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877292.669541:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877292.669573:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511355136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877292.669595:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877292.669598:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877292.669599:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877292.669602:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2279 00000100:00080000:3.0:1788877292.669604:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877292.669607:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877292.669615:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877292.669624:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511355264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877292.669631:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511355136:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877292.669634:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877292.669641:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877292.669642:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877292.669645:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2279 00000100:00080000:3.0:1788877292.669648:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877292.669650:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877292.669653:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877292.669661:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511355392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877292.669668:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877292.669670:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877292.669672:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877292.669674:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2279 00000100:00080000:3.0:1788877292.669677:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877292.669680:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877292.669683:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877292.669690:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511355520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788877292.669694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877292.669702:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877292.669705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877292.669707:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877292.669710:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2279 00000100:00080000:3.0:1788877292.669712:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877292.669715:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877292.669718:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877292.669726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511355648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877292.669730:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877292/real 1788877292] req@ffffa06188873480 x1875771511355136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877292.669754:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511355136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877292.669762:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877292.669765:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877292.669767:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511355136:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877292.669786:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877292.669802:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877292.669810:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877292/real 1788877292] req@ffffa06188872300 x1875771511355264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877292.669818:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511355264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877292.669824:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877292.669826:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877292.669828:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877292.669831:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355392:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877292.669841:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877292.669847:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877292/real 1788877292] req@ffffa06188873b80 x1875771511355392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877292.669855:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511355392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877292.669864:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877292.669866:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877292.669868:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355392:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877292.669871:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877292.669881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877292.669887:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877292/real 1788877292] req@ffffa06188870700 x1875771511355520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877292.669893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511355520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877292.669898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877292.669900:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877292.669902:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877292.669905:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355648:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877292.669914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877292.669920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877292/real 1788877292] req@ffffa06188872680 x1875771511355648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877292.669927:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511355648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877292.669932:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877292.669934:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877292.669936:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355648:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877293.309483:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877293.565473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877294.333566:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877294.589462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877295.613470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877296.381479:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877296.637500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877297.405527:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877297.661485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877297.789500:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877297.789511:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877297.789515:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877297.789525:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2284 00000100:00080000:1.0:1788877297.789535:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877297.789542:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877297.789581:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511355776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877297.789609:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877297.789611:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877297.789613:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877297.789616:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2284 00000100:00080000:1.0:1788877297.789619:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877297.789622:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877297.789631:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877297.789642:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511355904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877297.789652:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511355776:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877297.789657:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877297.789661:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877297.789663:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877297.789666:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2284 00000100:00080000:1.0:1788877297.789668:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877297.789670:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877297.789673:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877297.789683:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511356032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877297.789689:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877297.789692:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877297.789693:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877297.789695:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2284 00000100:00080000:1.0:1788877297.789698:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877297.789700:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877297.789703:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877297.789710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511356160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877297.789716:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877297.789718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877297.789720:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788877297.789723:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877297.789725:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2284 00000100:00080000:1.0:1788877297.789729:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877297.789732:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877297.789736:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877297.789744:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511356288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877297.789761:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877297/real 1788877297] req@ffffa06188699180 x1875771511355776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877297.789776:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511355776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877297.789783:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877297.789786:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877297.789789:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511355776:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877297.789808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877297.789824:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877297.789832:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877297/real 1788877297] req@ffffa0618869b800 x1875771511355904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877297.789840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511355904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877297.789846:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877297.789848:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877297.789850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511355904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877297.789853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356032:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877297.789865:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877297.789871:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877297/real 1788877297] req@ffffa0618869aa00 x1875771511356032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877297.789878:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511356032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877297.789883:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877297.789885:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877297.789886:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356032:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877297.789890:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877297.789902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877297.789908:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877297/real 1788877297] req@ffffa0618869b100 x1875771511356160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877297.789915:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511356160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877297.789920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877297.789922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877297.789924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877297.789927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356288:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877297.789936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877297.789941:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877297/real 1788877297] req@ffffa06188698380 x1875771511356288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877297.789953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511356288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877297.789958:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877297.789960:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877297.789961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356288:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877298.685484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877299.453497:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877299.709513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877300.477569:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877300.733508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877301.757475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877302.525537:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877302.781494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877302.909522:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877302.909530:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877302.909535:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877302.909543:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2289 00000100:00080000:3.0:1788877302.909555:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877302.909562:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877302.909596:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511356416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877302.909618:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877302.909621:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877302.909622:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877302.909625:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2289 00000100:00080000:3.0:1788877302.909627:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877302.909630:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877302.909637:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877302.909646:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511356544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877302.909654:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877302.909656:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877302.909658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877302.909660:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2289 00000100:00080000:3.0:1788877302.909662:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877302.909664:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877302.909667:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877302.909673:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511356672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877302.909681:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511356416:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877302.909683:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877302.909689:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877302.909693:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877302.909696:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2289 00000100:00080000:3.0:1788877302.909698:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877302.909700:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877302.909703:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877302.909712:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511356800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877302.909717:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877302.909720:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877302.909721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877302.909723:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2289 00000100:00080000:3.0:1788877302.909736:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877302.909739:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877302.909742:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877302.909751:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511356928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877302.909752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877302.909796:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877302/real 1788877302] req@ffffa06188871c00 x1875771511356416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877302.909814:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511356416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877302.909824:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877302.909828:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877302.909832:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511356416:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877302.909840:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877302.909861:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877302.909872:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877302/real 1788877302] req@ffffa06188871880 x1875771511356544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877302.909883:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511356544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877302.909898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877302.909901:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877302.909905:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877302.909927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356672:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877302.909945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877302.909955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877302/real 1788877302] req@ffffa06188870a80 x1875771511356672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877302.909966:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511356672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877302.909974:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877302.909977:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877302.909980:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356672:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877302.909985:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877302.910001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877302.910011:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877302/real 1788877302] req@ffffa06188870380 x1875771511356800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877302.910022:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511356800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877302.910029:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877302.910032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877302.910036:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877302.910042:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356928:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877302.910058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877302.910067:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877302/real 1788877302] req@ffffa06188872d80 x1875771511356928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877302.910077:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511356928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877302.910085:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877302.910088:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877302.910091:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511356928:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877303.549519:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877303.805488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877304.829479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877305.597461:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877305.853470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877306.621556:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877306.877514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877307.901474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877308.029724:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877308.029730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877308.029734:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877308.029742:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2294 00000100:00080000:3.0:1788877308.029749:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877308.029755:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877308.029786:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511357056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877308.029809:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877308.029812:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877308.029814:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788877308.029847:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511357056:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877308.029872:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2294 00000100:00080000:3.0:1788877308.029876:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877308.029879:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877308.029887:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877308.029900:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511357184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877308.029908:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877308.029912:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877308.029913:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877308.029916:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2294 00000100:00080000:3.0:1788877308.029919:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877308.029922:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877308.029927:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877308.029933:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511357312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877308.029940:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357184:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788877308.029946:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877308.029950:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877308.029951:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877308.029955:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2294 00000100:00080000:3.0:1788877308.029957:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877308.029960:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877308.029965:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877308.029975:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511357440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877308.029978:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357312:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788877308.029993:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877308.029996:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877308.029998:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877308.030000:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2294 00000100:00080000:3.0:1788877308.030005:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877308.030007:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877308.030011:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877308.030021:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511357568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877308.030024:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877308.030054:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357568:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877308.669507:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877308.925522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877309.693562:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877309.949472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877310.973457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877311.101671:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877311.101712:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877311.101728:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877311.101769:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877311.101788:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877311.101800:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877311.101811:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788877311.101822:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877308/real 1788877311] req@ffffa06188871880 x1875771511357440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877311.101849:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511357440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877311.101862:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877311.101866:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877311.101871:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877311.101879:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877308/real 1788877311] req@ffffa06188870a80 x1875771511357312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877311.101893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511357312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877311.101908:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877311.101912:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877311.101915:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357312:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877311.101922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877308/real 1788877311] req@ffffa06188870380 x1875771511357184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877311.101931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511357184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877311.101939:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877311.101942:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877311.101946:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877311.101953:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877308/real 1788877311] req@ffffa06188872d80 x1875771511357056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877311.101967:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511357056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877311.101975:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877311.101978:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877311.101981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511357056:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877311.102008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877308/real 1788877311] req@ffffa06188871c00 x1875771511357568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877311.102020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511357568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877311.102028:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877311.102032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877311.102041:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357568:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877311.741543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877311.997479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877312.765556:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877313.021498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877313.149484:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877313.149492:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877313.149495:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877313.149504:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2299 00000100:00080000:1.0:1788877313.149512:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877313.149518:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877313.149549:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511357696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877313.149572:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877313.149575:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877313.149576:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877313.149579:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2299 00000100:00080000:1.0:1788877313.149582:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877313.149584:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877313.149593:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877313.149604:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511357824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877313.149612:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877313.149614:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877313.149616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877313.149621:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2299 00000100:00100000:2.0:1788877313.149621:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511357696:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877313.149623:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877313.149630:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877313.149635:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877313.149642:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511357952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877313.149649:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877313.149651:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877313.149653:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877313.149655:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2299 00000100:00080000:1.0:1788877313.149657:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877313.149660:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877313.149664:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877313.149670:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511358080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877313.149677:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877313.149679:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877313.149681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877313.149689:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2299 00000800:00000100:2.0:1788877313.149690:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877313.149697:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877313.149703:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877313.149709:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877313.149722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511358208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877313.149729:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877313/real 1788877313] req@ffffa06188699c00 x1875771511357696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877313.149750:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511357696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877313.149762:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877313.149766:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877313.149769:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511357696:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877313.149790:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877313.149811:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877313.149822:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877313/real 1788877313] req@ffffa06188698380 x1875771511357824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877313.149834:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511357824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877313.149842:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877313.149845:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877313.149849:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877313.149856:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357952:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877313.149872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877313.149883:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877313/real 1788877313] req@ffffa0618869b100 x1875771511357952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877313.149893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511357952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877313.149901:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877313.149904:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877313.149907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511357952:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877313.149913:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877313.149930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877313.149939:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877313/real 1788877313] req@ffffa0618869aa00 x1875771511358080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877313.149950:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511358080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877313.149958:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877313.149961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877313.149965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877313.149971:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358208:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877313.149986:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877313.149996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877313/real 1788877313] req@ffffa0618869b800 x1875771511358208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877313.150008:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511358208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877313.150016:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877313.150019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877313.150022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358208:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877314.045544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877314.813563:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877315.069486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877315.837549:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877316.093493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877317.117526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877317.885549:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877318.141494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877318.269505:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877318.269517:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877318.269521:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877318.269530:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2304 00000100:00080000:3.0:1788877318.269541:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877318.269550:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877318.269583:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511358336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877318.269605:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877318.269608:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877318.269610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877318.269613:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2304 00000100:00080000:3.0:1788877318.269616:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877318.269618:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877318.269625:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877318.269636:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511358464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877318.269643:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877318.269646:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877318.269647:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877318.269650:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2304 00000100:00080000:3.0:1788877318.269652:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877318.269654:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877318.269658:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877318.269665:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511358592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877318.269672:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877318.269675:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877318.269676:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877318.269682:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2304 00000100:00080000:3.0:1788877318.269686:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877318.269693:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511358336:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877318.269694:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877318.269699:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877318.269716:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511358720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877318.269723:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877318.269726:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877318.269727:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877318.269729:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2304 00000100:00080000:3.0:1788877318.269731:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877318.269734:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877318.269737:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877318.269744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511358848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877318.269798:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877318.269860:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877318/real 1788877318] req@ffffa06188872680 x1875771511358336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877318.269888:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511358336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877318.269901:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877318.269907:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877318.269911:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511358336:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877318.269921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877318.269946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877318.269960:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877318/real 1788877318] req@ffffa06188870700 x1875771511358464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877318.269972:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511358464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877318.269987:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877318.269991:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877318.269995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877318.270026:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358592:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877318.270045:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877318.270056:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877318/real 1788877318] req@ffffa06188873b80 x1875771511358592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877318.270068:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511358592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877318.270077:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877318.270081:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877318.270084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358592:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877318.270091:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877318.270110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877318.270135:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877318/real 1788877318] req@ffffa06188872300 x1875771511358720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877318.270149:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511358720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877318.270157:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877318.270161:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877318.270165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877318.270171:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358848:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877318.270191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877318.270202:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877318/real 1788877318] req@ffffa06188873480 x1875771511358848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877318.270213:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511358848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877318.270221:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877318.270224:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877318.270227:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511358848:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877318.909630:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877319.165490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877320.189512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877320.957480:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877321.213473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877321.981547:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877322.237529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877323.261464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877323.389492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877323.389504:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877323.389508:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877323.389528:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2310 00000100:00080000:3.0:1788877323.389539:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877323.389547:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877323.389584:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511358976/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877323.389613:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877323.389618:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877323.389620:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877323.389625:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2310 00000100:00080000:3.0:1788877323.389630:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877323.389635:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877323.389645:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877323.389657:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511358976:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877323.389663:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511359104/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877323.389679:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877323.389682:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877323.389685:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877323.389690:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2310 00000100:00080000:3.0:1788877323.389702:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877323.389707:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788877323.389713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788877323.389714:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877323.389732:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511359232/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877323.389744:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877323.389745:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877323/real 1788877323] req@ffffa06188871f80 x1875771511358976/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877323.389755:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877323.389760:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877323.389767:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2310 00000100:00100000:2.0:1788877323.389770:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511358976/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877323.389775:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877323.389785:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877323.389785:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:2.0:1788877323.389793:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788877323.389797:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877323.389798:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511358976:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877323.389817:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511359360/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877323.389830:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359104:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788877323.389833:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877323.389841:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877323.389843:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877323.389849:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2310 00000100:00080000:3.0:1788877323.389857:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877323.389863:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788877323.389866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788877323.389872:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877323.389886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877323/real 1788877323] req@ffffa06188872a00 x1875771511359104/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877323.389891:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511359488/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877323.389917:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511359104/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877323.389925:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877323.389928:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877323.389934:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877323.389942:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359232:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877323.389965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877323.389978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877323/real 1788877323] req@ffffa06188873100 x1875771511359232/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877323.389988:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511359232/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877323.389994:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877323.389996:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877323.389998:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359232:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877323.390010:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877323.390026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877323.390033:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877323/real 1788877323] req@ffffa06188871500 x1875771511359360/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877323.390040:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511359360/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877323.390046:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877323.390048:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877323.390051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877323.390055:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359488:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877323.390064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877323.390070:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877323/real 1788877323] req@ffffa06188873800 x1875771511359488/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877323.390077:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511359488/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877323.390082:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877323.390084:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877323.390106:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359488:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877324.029507:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877324.285475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877325.053540:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877325.309462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877326.333483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877327.101536:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877327.357443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877328.125633:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877328.381515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877328.509478:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877328.509491:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877328.509497:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877328.509510:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2315 00000100:00080000:3.0:1788877328.509523:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877328.509533:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877328.509585:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511359616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877328.509617:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877328.509621:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877328.509623:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877328.509628:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2315 00000100:00080000:3.0:1788877328.509633:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877328.509647:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877328.509659:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877328.509666:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511359616:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877328.509678:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511359744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877328.509690:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877328.509694:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877328.509696:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877328.509700:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2315 00000100:00080000:3.0:1788877328.509704:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877328.509708:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877328.509713:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877328.509726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511359872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877328.509736:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877328.509740:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877328.509742:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788877328.509749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877328.509749:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2315 00000100:00080000:3.0:1788877328.509757:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877328.509762:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877328.509768:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877328.509780:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511360000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877328.509791:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877328/real 1788877328] req@ffffa06188873800 x1875771511359616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877328.509811:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877328.509816:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877328.509818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877328.509821:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511359616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877328.509826:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2315 00000100:00080000:3.0:1788877328.509835:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877328.509839:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877328.509841:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:2.0:1788877328.509845:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788877328.509850:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877328.509851:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511359616:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877328.509867:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511360128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877328.509890:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877328.509909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877328.509919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877328/real 1788877328] req@ffffa06188871500 x1875771511359744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877328.509931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511359744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877328.509940:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877328.509944:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877328.509948:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877328.509955:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359872:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877328.509976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877328.509988:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877328/real 1788877328] req@ffffa06188873100 x1875771511359872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877328.509998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511359872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877328.510004:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877328.510007:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877328.510009:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511359872:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877328.510013:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877328.510036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877328.510044:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877328/real 1788877328] req@ffffa06188872a00 x1875771511360000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877328.510052:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511360000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877328.510058:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877328.510060:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877328.510062:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877328.510065:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360128:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877328.510093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877328.510101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877328/real 1788877328] req@ffffa06188871f80 x1875771511360128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877328.510109:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511360128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877328.510115:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877328.510118:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877328.510122:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360128:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877329.405444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877330.173569:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877330.429528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877331.197591:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877331.453473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877332.477494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877333.245554:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877333.501498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877333.629495:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877333.629509:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877333.629514:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877333.629525:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2320 00000100:00080000:2.0:1788877333.629537:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877333.629545:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877333.629590:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511360256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877333.629619:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877333.629623:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877333.629624:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877333.629628:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2320 00000100:00080000:2.0:1788877333.629630:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877333.629633:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877333.629642:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877333.629655:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511360384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877333.629662:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877333.629665:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877333.629674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877333.629678:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2320 00000100:00080000:2.0:1788877333.629685:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788877333.629687:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511360256:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877333.629690:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877333.629707:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877333.629722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511360512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877333.629731:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877333.629734:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877333.629735:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877333.629739:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2320 00000100:00080000:2.0:1788877333.629744:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877333.629748:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877333.629752:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877333.629761:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511360640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788877333.629769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877333.629783:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877333.629786:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877333.629788:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877333.629794:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2320 00000100:00080000:2.0:1788877333.629798:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877333.629801:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877333.629806:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877333.629817:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511360768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877333.629821:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877333/real 1788877333] req@ffffa061888ad500 x1875771511360256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877333.629873:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511360256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877333.629883:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877333.629887:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877333.629891:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511360256:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877333.629898:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877333.629914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877333.629922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877333/real 1788877333] req@ffffa061888af480 x1875771511360384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877333.629929:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511360384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877333.629935:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877333.629937:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877333.629939:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877333.629959:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360512:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788877333.629971:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877333.629978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877333/real 1788877333] req@ffffa061888adf80 x1875771511360512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877333.629985:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511360512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877333.629991:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877333.629993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877333.629994:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360512:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877333.629999:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877333.630017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877333.630025:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877333/real 1788877333] req@ffffa061888ac000 x1875771511360640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877333.630032:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511360640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877333.630037:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877333.630040:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877333.630042:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877333.630059:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360768:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788877333.630074:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877333.630082:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877333/real 1788877333] req@ffffa061888ae300 x1875771511360768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877333.630094:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511360768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877333.630099:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877333.630102:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877333.630104:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511360768:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877334.269558:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877334.525459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877335.549456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877336.317484:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877336.573502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877337.341571:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877337.597537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877338.621503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877338.749511:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877338.749521:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877338.749526:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877338.749536:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2325 00000100:00080000:1.0:1788877338.749546:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877338.749555:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877338.749593:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511360896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877338.749623:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877338.749627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877338.749629:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877338.749634:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2325 00000100:00080000:1.0:1788877338.749639:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877338.749644:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877338.749654:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877338.749660:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511360896:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877338.749672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511361024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877338.749686:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877338.749689:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877338.749691:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877338.749696:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2325 00000100:00080000:1.0:1788877338.749701:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877338.749706:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877338.749712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1788877338.749716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877338.749729:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511361152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877338.749741:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877338.749745:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877338.749748:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877338.749749:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877338/real 1788877338] req@ffffa06188699180 x1875771511360896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877338.749757:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2325 00000100:00080000:1.0:1788877338.749762:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877338.749771:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877338.749777:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877338.749779:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511360896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877338.749790:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788877338.749795:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511361280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877338.749796:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877338.749804:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511360896:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877338.749822:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877338.749826:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877338.749829:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877338.749831:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361024:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788877338.749836:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2325 00000100:00080000:1.0:1788877338.749842:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877338.749847:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877338.749853:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877338.749858:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877338.749874:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877338/real 1788877338] req@ffffa06188699f80 x1875771511361024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877338.749879:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511361408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877338.749909:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511361024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877338.749929:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877338.749934:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877338.749937:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877338.749943:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361152:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877338.749963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877338.749976:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877338/real 1788877338] req@ffffa06188698000 x1875771511361152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877338.749988:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511361152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877338.749997:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877338.750000:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877338.750004:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361152:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877338.750010:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877338.750046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877338.750060:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877338/real 1788877338] req@ffffa0618869ad80 x1875771511361280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877338.750070:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511361280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877338.750075:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877338.750077:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877338.750080:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877338.750096:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361408:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877338.750117:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877338.750129:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877338/real 1788877338] req@ffffa0618869bb80 x1875771511361408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877338.750142:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511361408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877338.750151:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877338.750154:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877338.750164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511361408:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877339.389505:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877339.645500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877340.413532:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877340.669459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877341.693486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877342.461498:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877342.717459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877343.485539:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877343.741507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877343.869508:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877343.869519:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877343.869523:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877343.869535:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2330 00000100:00080000:3.0:1788877343.869545:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877343.869555:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877343.869590:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511361536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877343.869622:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877343.869627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877343.869631:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877343.869636:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2330 00000100:00080000:3.0:1788877343.869641:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877343.869647:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877343.869658:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877343.869677:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511361664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877343.869692:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877343.869697:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877343.869699:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877343.869705:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2330 00000100:00080000:3.0:1788877343.869710:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877343.869716:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877343.869723:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877343.869739:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511361792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877343.869752:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877343.869756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877343.869758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877343.869765:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2330 00000100:00080000:3.0:1788877343.869772:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877343.869848:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877343.869857:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877343.869878:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511361920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877343.869914:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877343.869920:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877343.869926:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877343.869936:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2330 00000100:00080000:3.0:1788877343.869942:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877343.869951:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877343.869959:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877343.869976:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511362048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877344.765463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877345.533499:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877345.789505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877346.557592:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877346.813464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877347.837478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877348.605481:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877348.861460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877348.989496:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877348.989506:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877348.989510:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877348.989520:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2335 00000100:00080000:1.0:1788877348.989533:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877348.989540:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877348.989579:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511362176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877348.989606:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877348.989609:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877348.989611:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877348.989616:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2335 00000100:00080000:1.0:1788877348.989621:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877348.989626:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877348.989635:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877348.989653:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511362304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877348.989669:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877348.989673:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877348.989675:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877348.989679:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2335 00000100:00080000:1.0:1788877348.989683:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877348.989687:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877348.989693:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877348.989704:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511362432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877348.989720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877348.989724:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877348.989726:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877348.989731:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2335 00000100:00080000:1.0:1788877348.989735:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877348.989739:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877348.989745:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877348.989760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511362560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877348.989789:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877348.989797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877348.989801:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877348.989812:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2335 00000100:00080000:1.0:1788877348.989817:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877348.989821:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877348.989825:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877348.989840:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511362688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1788877349.629605:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877349.885529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877350.909504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877351.677490:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877351.933542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877352.701708:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877352.957517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877353.981499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877354.109480:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877354.109508:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877354.109514:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877354.109539:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2340 00000100:00080000:1.0:1788877354.109569:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877354.109580:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877354.109626:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511362816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877354.109659:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877354.109663:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877354.109666:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877354.109671:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2340 00000100:00080000:1.0:1788877354.109676:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877354.109698:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877354.109709:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877354.109733:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511362944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877354.109751:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877354.109757:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877354.109761:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877354.109766:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2340 00000100:00080000:1.0:1788877354.109815:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877354.109822:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877354.109833:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877354.109852:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511363072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877354.109882:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877354.109887:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877354.109893:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877354.109904:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2340 00000100:00080000:1.0:1788877354.109914:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877354.109919:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877354.109925:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877354.109972:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511363200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877354.110035:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877354.110041:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877354.110047:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877354.110061:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2340 00000100:00080000:1.0:1788877354.110067:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877354.110071:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877354.110077:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877354.110101:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511363328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1788877354.749528:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877355.005506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877355.773569:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877356.029529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877357.053563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877357.821574:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877358.077506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877358.845615:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877359.101510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877359.229616:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511363456:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877359.229678:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877359.229714:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877359/real 1788877359] req@ffffa06198aff480 x1875771511363456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877359.229735:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511363456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877359.229748:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877359.229751:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877359.229754:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511363456:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877359.229772:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877359.229786:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877359.229794:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877359/real 1788877359] req@ffffa06198afdc00 x1875771511363584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877359.229802:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511363584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877359.229807:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877359.229809:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877359.229811:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877359.229814:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363712:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877359.229824:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877359.229829:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877359/real 1788877359] req@ffffa06198afc700 x1875771511363712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877359.229836:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511363712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877359.229841:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877359.229843:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877359.229845:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363712:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877359.229849:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877359.229859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877359.229864:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877359/real 1788877359] req@ffffa06198afc000 x1875771511363840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877359.229871:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511363840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877359.229876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877359.229878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877359.229880:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877359.229883:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363968:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877359.229908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877359.229919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877359/real 1788877359] req@ffffa06198afea00 x1875771511363968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877359.229947:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511363968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877359.229955:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877359.229959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877359.229962:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511363968:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877360.125459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877360.893564:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877361.149533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877361.917557:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877362.173542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877363.197490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877363.965483:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877364.221516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877364.349823:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511364096:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877364.349984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877364.350057:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877364/real 1788877364] req@ffffa06198afce00 x1875771511364096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877364.350098:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afce00 x1875771511364096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877364.350142:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877364.350152:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877364.350170:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511364096:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877364.350216:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877364.350256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877364.350272:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877364/real 1788877364] req@ffffa06198afe300 x1875771511364224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877364.350299:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511364224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877364.350321:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877364.350330:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877364.350340:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877364.350356:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364352:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877364.350467:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877364.350507:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877364/real 1788877364] req@ffffa06198afca80 x1875771511364352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877364.350523:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511364352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877364.350538:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877364.350542:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877364.350547:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364352:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877364.350555:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877364.350575:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877364.350584:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877364/real 1788877364] req@ffffa06198afd500 x1875771511364480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877364.350593:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511364480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877364.350608:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877364.350611:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877364.350614:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877364.350630:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364608:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877364.350645:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877364.350654:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877364/real 1788877364] req@ffffa06198afd180 x1875771511364608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877364.350663:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511364608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877364.350670:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877364.350673:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877364.350676:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364608:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877364.989532:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877365.245533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877366.269491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877367.037555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877367.293553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877368.061590:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877368.317526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877369.341506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877369.469640:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511364736:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877369.469708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877369.469824:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877369/real 1788877369] req@ffffa06198afc380 x1875771511364736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877369.469840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511364736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877369.469848:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877369.469852:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877369.469855:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511364736:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877369.469862:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877369.469903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877369.469917:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877369/real 1788877369] req@ffffa06198afd180 x1875771511364864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877369.469933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511364864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877369.469943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877369.469947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877369.469951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877369.469976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364992:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877369.469994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877369.470004:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877369/real 1788877369] req@ffffa06198afd500 x1875771511364992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877369.470013:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511364992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877369.470019:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877369.470021:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877369.470024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511364992:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877369.470028:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511365120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877369.470046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877369.470053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877369/real 1788877369] req@ffffa06198afca80 x1875771511365120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877369.470061:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511365120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877369.470066:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877369.470068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877369.470070:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511365120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877369.470074:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511365248:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877369.470084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877369.470090:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877369/real 1788877369] req@ffffa06198afe300 x1875771511365248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877369.470097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511365248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877369.470103:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877369.470105:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877369.470108:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511365248:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877370.109559:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877370.365477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877371.133706:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877371.389468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877372.413598:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877373.181562:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877373.437522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877374.205581:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877374.461547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877374.589528:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877374.589536:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877374.589541:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877374.589551:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2361 00000100:00080000:2.0:1788877374.589563:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877374.589570:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877374.589602:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511365376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877374.589629:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877374.589632:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877374.589633:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877374.589636:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2361 00000100:00080000:2.0:1788877374.589638:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877374.589642:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877374.589650:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877374.589659:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511365504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877374.589670:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877374.589678:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877374.589682:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877374.589691:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2361 00000100:00080000:2.0:1788877374.589696:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877374.589699:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877374.589702:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877374.589713:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511365632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877374.589721:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877374.589724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877374.589725:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877374.589728:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2361 00000100:00080000:2.0:1788877374.589730:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877374.589732:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877374.589736:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877374.589742:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511365760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877374.589750:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877374.589753:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877374.589755:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877374.589757:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2361 00000100:00080000:2.0:1788877374.589759:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877374.589762:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877374.589764:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877374.589771:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511365888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877375.485511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877376.253524:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877376.509551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877377.277610:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877377.533495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788877377.661819:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788877377.661898:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788877377.661925:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877377.661990:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877377.662004:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877377.662019:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877377.662033:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:1.0:1788877378.557513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877379.325580:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877379.581506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877379.709590:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877379.709599:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877379.709603:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877379.709615:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2366 00000100:00080000:2.0:1788877379.709628:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877379.709637:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877379.709674:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511366016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877379.709705:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877379.709708:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877379.709711:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877379.709717:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2366 00000100:00080000:2.0:1788877379.709720:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877379.709725:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877379.709735:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877379.709751:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511366144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877379.709759:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511366016:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877379.709766:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877379.709771:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877379.709773:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877379.709778:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2366 00000100:00080000:2.0:1788877379.709780:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877379.709785:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877379.709791:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877379.709803:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511366272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877379.709812:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877379.709816:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877379.709817:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877379.709821:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2366 00000100:00080000:2.0:1788877379.709825:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877379.709866:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877379.709876:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877379.709896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877379.709906:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511366400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877379.709932:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877379.709937:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877379.709941:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877379.709950:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2366 00000100:00080000:2.0:1788877379.709959:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877379.709967:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877379.709976:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877379.709978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877379/real 1788877379] req@ffffa061888ad880 x1875771511366016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877379.710004:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511366528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877379.710028:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511366016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877379.710061:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877379.710069:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877379.710079:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511366016:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877379.710142:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877379.710212:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877379.710252:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877379/real 1788877379] req@ffffa061888ac380 x1875771511366144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877379.710287:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511366144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877379.710306:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877379.710314:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877379.710323:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877379.710340:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366272:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877379.710430:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877379.710464:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877379/real 1788877379] req@ffffa061888ad180 x1875771511366272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877379.710489:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511366272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877379.710502:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877379.710507:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877379.710512:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366272:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877379.710522:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877379.712327:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877379.712368:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877379/real 1788877379] req@ffffa061888ac700 x1875771511366400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877379.712385:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511366400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877379.712394:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877379.712422:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877379.712425:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877379.712432:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366528:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877379.712468:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877379.712483:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877379/real 1788877379] req@ffffa061888aed80 x1875771511366528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877379.712508:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511366528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877379.712524:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877379.712528:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877379.712532:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366528:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877380.349581:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877380.605647:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877381.629494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877382.397475:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877382.653521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877383.421672:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877383.677505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877384.701488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877384.829525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877384.829534:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877384.829538:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877384.829547:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2371 00000100:00080000:1.0:1788877384.829556:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877384.829574:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877384.829606:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511366656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877384.829628:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877384.829632:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877384.829634:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877384.829638:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2371 00000100:00080000:1.0:1788877384.829642:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877384.829646:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877384.829656:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877384.829680:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511366784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877384.829680:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511366656:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877384.829691:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877384.829694:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877384.829696:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877384.829699:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2371 00000100:00080000:1.0:1788877384.829702:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877384.829706:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877384.829711:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877384.829720:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511366912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877384.829729:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877384.829742:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877384.829744:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877384.829747:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2371 00000100:00080000:1.0:1788877384.829750:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877384.829753:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877384.829758:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877384.829759:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877384.829766:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511367040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877384.829774:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877384.829777:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877384.829779:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877384.829783:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2371 00000100:00080000:1.0:1788877384.829786:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877384.829790:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877384.829791:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877384/real 1788877384] req@ffffa06188699c00 x1875771511366656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788877384.829794:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877384.829815:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511367168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877384.829825:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511366656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877384.829835:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877384.829839:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877384.829854:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511366656:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877384.829875:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877384.829894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877384.829903:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877384/real 1788877384] req@ffffa06188698380 x1875771511366784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877384.829911:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511366784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877384.829921:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877384.829923:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877384.829925:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877384.829929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366912:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877384.829939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877384.829945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877384/real 1788877384] req@ffffa0618869b100 x1875771511366912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877384.829952:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511366912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877384.829957:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877384.829959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877384.829960:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511366912:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877384.829964:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877384.829974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877384.829979:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877384/real 1788877384] req@ffffa0618869aa00 x1875771511367040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877384.829986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511367040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877384.829991:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877384.829993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877384.829995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877384.829999:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367168:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877384.830008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877384.830013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877384/real 1788877384] req@ffffa0618869b800 x1875771511367168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877384.830020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511367168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877384.830025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877384.830027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877384.830029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367168:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788877385.469534:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877385.725597:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877386.493728:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877386.749483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877387.773545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877388.541507:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877388.797488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877389.565652:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877389.821475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877389.949514:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877389.949525:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877389.949530:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877389.949542:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2376 00000100:00080000:1.0:1788877389.949554:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877389.949562:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877389.949600:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511367296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877389.949634:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877389.949638:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877389.949640:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877389.949644:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2376 00000100:00080000:1.0:1788877389.949648:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877389.949652:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877389.949661:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877389.949673:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511367424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877389.949685:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877389.949689:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877389.949692:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877389.949695:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2376 00000100:00080000:1.0:1788877389.949699:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877389.949703:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877389.949709:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877389.949735:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511367296:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877389.949740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511367552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877389.949765:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877389.949769:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877389.949771:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877389.949776:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2376 00000100:00080000:1.0:1788877389.949804:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877389.949811:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877389.949818:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877389.949838:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511367680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877389.949852:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877389.949856:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877389.949858:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877389.949863:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2376 00000100:00080000:1.0:1788877389.949867:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877389.949876:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788877389.949882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788877389.949885:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877389.949902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511367808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877389.949957:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877389/real 1788877389] req@ffffa06188699180 x1875771511367296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877389.950006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511367296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877389.950023:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877389.950030:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877389.950036:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511367296:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877389.950048:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877389.950087:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877389.950108:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877389/real 1788877389] req@ffffa06188699f80 x1875771511367424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877389.950130:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511367424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877389.950143:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877389.950149:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877389.950154:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877389.950189:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367552:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877389.950221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877389.950239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877389/real 1788877389] req@ffffa06188698000 x1875771511367552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877389.950276:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511367552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877389.950294:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877389.950300:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877389.950307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367552:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877389.950320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877389.950358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877389.950382:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877389/real 1788877389] req@ffffa0618869ad80 x1875771511367680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877389.950483:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511367680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877389.950497:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877389.950503:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877389.950508:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877389.950518:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367808:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877389.950554:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877389.950574:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877389/real 1788877389] req@ffffa0618869bb80 x1875771511367808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877389.950593:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511367808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877389.950605:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877389.950610:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877389.950615:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511367808:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877390.845485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877391.613619:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877391.869540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877392.637622:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877392.893490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877393.917507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877394.685498:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877394.941508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877395.069595:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877395.069620:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877395.069625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877395.069637:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2381 00000100:00080000:2.0:1788877395.069648:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877395.069656:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877395.069697:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511367936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877395.069729:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877395.069733:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877395.069735:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877395.069739:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2381 00000100:00080000:2.0:1788877395.069744:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877395.069748:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877395.069774:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877395.069786:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511367936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877395.069816:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511368064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877395.069834:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877395.069839:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877395.069842:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877395.069848:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2381 00000100:00080000:2.0:1788877395.069874:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877395.069882:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788877395.069888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788877395.069899:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877395.069923:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511368192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877395.069935:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877395/real 1788877395] req@ffffa061888ace00 x1875771511367936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877395.069938:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877395.069958:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877395.069965:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877395.069975:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2381 00000100:00100000:3.0:1788877395.069980:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511367936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877395.069984:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877395.069999:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788877395.070001:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877395.070010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:2.0:1788877395.070014:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877395.070016:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511367936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877395.070048:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877395.070074:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511368320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788877395.070089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877395.070107:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877395.070112:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877395.070114:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788877395.070116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877395/real 1788877395] req@ffffa061888aea00 x1875771511368064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877395.070128:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2381 00000100:00080000:2.0:1788877395.070143:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788877395.070150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511368064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877395.070151:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788877395.070168:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788877395.070174:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788877395.070174:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877395.070183:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877395.070203:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368192:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877395.070204:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511368448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788877395.070237:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877395.070248:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877395/real 1788877395] req@ffffa061888aed80 x1875771511368192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877395.070257:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511368192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877395.070263:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877395.070266:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877395.070268:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368192:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877395.070280:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877395.070293:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877395.070299:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877395/real 1788877395] req@ffffa061888ac700 x1875771511368320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877395.070307:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511368320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877395.070313:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877395.070315:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877395.070318:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877395.070321:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368448:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877395.070331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877395.070336:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877395/real 1788877395] req@ffffa061888ad180 x1875771511368448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877395.070352:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511368448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877395.070356:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877395.070359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877395.070361:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368448:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877395.709627:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877395.965458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877396.989502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788877397.757548:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877398.013465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877398.781559:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877399.037477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877400.061581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877400.189590:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877400.189623:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877400.189629:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877400.189645:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2386 00000100:00080000:2.0:1788877400.189664:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877400.189674:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877400.189764:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511368576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877400.189797:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877400.189802:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877400.189804:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877400.189809:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2386 00000100:00080000:2.0:1788877400.189813:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877400.189817:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877400.189829:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877400.189844:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511368704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877400.189856:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877400.189860:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877400.189863:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877400.189867:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2386 00000100:00080000:2.0:1788877400.189871:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877400.189875:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877400.189880:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877400.189888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511368576:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877400.189897:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511368832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877400.189910:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877400.189915:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877400.189917:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877400.189920:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2386 00000100:00080000:2.0:1788877400.189924:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877400.189928:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877400.189933:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877400.189949:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511368960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877400.189959:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877400.189963:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877400.189965:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877400.189968:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2386 00000100:00080000:2.0:1788877400.189972:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877400.189977:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877400.189982:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877400.189995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877400.189998:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511369088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877400.190066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877400/real 1788877400] req@ffffa061888ac380 x1875771511368576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877400.190110:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511368576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877400.190136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877400.190144:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877400.190153:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511368576:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877400.190166:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877400.190228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877400.190246:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877400/real 1788877400] req@ffffa061888ad880 x1875771511368704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877400.190260:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511368704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877400.190270:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877400.190273:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877400.190277:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877400.190307:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368832:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877400.190328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877400.190338:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877400/real 1788877400] req@ffffa061888aca80 x1875771511368832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877400.190351:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511368832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877400.190367:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877400.190371:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877400.190374:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368832:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877400.190380:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877400.190436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877400.190453:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877400/real 1788877400] req@ffffa061888adc00 x1875771511368960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877400.190466:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511368960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877400.190475:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877400.190479:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877400.190482:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511368960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877400.190488:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369088:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877400.190505:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877400.190515:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877400/real 1788877400] req@ffffa061888af100 x1875771511369088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877400.190527:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511369088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877400.190541:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877400.190544:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877400.190548:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369088:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877401.085448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877401.853804:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877402.109558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877403.133485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877404.157495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877404.925667:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877405.181501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877405.309585:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877405.309602:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877405.309610:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877405.309644:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2392 00000100:00080000:2.0:1788877405.309663:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877405.309676:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877405.309777:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511369216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877405.309825:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877405.309829:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877405.309831:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877405.309836:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2392 00000100:00080000:2.0:1788877405.309840:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877405.309844:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877405.309856:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877405.309872:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511369344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877405.309901:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511369216:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877405.309914:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877405.309920:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877405.309924:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877405.309932:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2392 00000100:00080000:2.0:1788877405.309938:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877405.309945:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877405.309954:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877405.309991:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511369472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877405.310002:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788877405.310013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877405.310015:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877405.310019:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877405.310025:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2392 00000100:00080000:2.0:1788877405.310031:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877405.310062:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877405.310070:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877405.310088:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511369600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877405.310099:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877405.310104:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877405.310106:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877405.310110:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2392 00000100:00080000:2.0:1788877405.310113:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877405.310116:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877405.310123:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877405/real 1788877405] req@ffffa061888afb80 x1875771511369216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788877405.310124:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877405.310168:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511369728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877405.310181:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511369216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877405.310198:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877405.310205:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877405.310214:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511369216:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877405.310251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877405.310283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877405.310299:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877405/real 1788877405] req@ffffa061888ae680 x1875771511369344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877405.310315:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511369344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877405.310326:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877405.310339:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877405.310343:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877405.310350:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369472:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877405.310370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877405.310429:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877405/real 1788877405] req@ffffa061888ae300 x1875771511369472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877405.310440:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511369472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877405.310447:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877405.310449:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877405.310451:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369472:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877405.310457:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877405.310501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877405.310515:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877405/real 1788877405] req@ffffa061888ac000 x1875771511369600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877405.310530:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511369600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877405.310539:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877405.310543:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877405.310547:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877405.310554:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369728:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877405.310573:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877405.310584:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877405/real 1788877405] req@ffffa061888adf80 x1875771511369728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877405.310597:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511369728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877405.310607:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877405.310611:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877405.310615:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369728:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877406.205520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877407.229457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877407.997494:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877408.253519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877409.277515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877410.301554:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877410.429519:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877410.429532:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877410.429537:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877410.429549:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2397 00000100:00080000:1.0:1788877410.429577:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877410.429597:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877410.429645:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511369856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877410.429679:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877410.429683:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877410.429685:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877410.429725:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2397 00000100:00080000:1.0:1788877410.429729:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877410.429732:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877410.429742:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877410.429758:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511369984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877410.429768:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877410.429772:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877410.429775:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788877410.429779:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511369856:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877410.429780:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2397 00000100:00080000:1.0:1788877410.429786:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877410.429794:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877410.429799:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877410.429808:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511370112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877410.429833:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877410.429836:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877410.429838:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877410.429840:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2397 00000100:00080000:1.0:1788877410.429843:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877410.429846:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877410.429850:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877410.429858:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511370240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877410.429865:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877410.429868:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877410.429869:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877410.429872:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2397 00000100:00080000:1.0:1788877410.429876:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877410.429878:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877410.429882:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877410.429890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877410.429892:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511370368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877410.429941:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877410/real 1788877410] req@ffffa0618869a680 x1875771511369856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877410.429966:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511369856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877410.429980:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877410.429985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877410.429990:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511369856:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877410.430019:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877410.430051:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877410.430069:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877410/real 1788877410] req@ffffa0618869bb80 x1875771511369984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877410.430087:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511369984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877410.430100:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877410.430104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877410.430109:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511369984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877410.430119:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511370112:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877410.430145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877410.430160:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877410/real 1788877410] req@ffffa0618869ad80 x1875771511370112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877410.430178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511370112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877410.430189:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877410.430194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877410.430209:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511370112:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877410.430218:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511370240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877410.430245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877410.430304:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877410/real 1788877410] req@ffffa06188698000 x1875771511370240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877410.430332:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511370240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877410.430350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877410.430363:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877410.430370:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511370240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877410.430457:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511370368:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877410.430487:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877410.430505:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877410/real 1788877410] req@ffffa06188699f80 x1875771511370368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877410.430541:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511370368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877410.430559:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877410.430573:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877410.430580:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511370368:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877411.069581:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877411.325496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877412.349509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877413.373551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877414.141533:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877414.397516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877415.421484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877415.549464:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877415.549472:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877415.549476:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877415.549485:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2402 00000100:00080000:1.0:1788877415.549495:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877415.549503:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877415.549539:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511370496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877415.549566:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877415.549569:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877415.549571:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877415.549575:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2402 00000100:00080000:1.0:1788877415.549579:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877415.549584:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877415.549593:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877415.549608:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511370624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877415.549619:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877415.549622:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877415.549624:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877415.549627:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2402 00000100:00080000:1.0:1788877415.549631:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877415.549635:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877415.549641:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877415.549652:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511370752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877415.549677:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877415.549681:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877415.549683:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877415.549687:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2402 00000100:00080000:1.0:1788877415.549690:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877415.549693:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877415.549698:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877415.549707:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511370880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877415.549714:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877415.549718:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877415.549722:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877415.549730:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2402 00000100:00080000:1.0:1788877415.549736:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877415.549739:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877415.549742:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877415.549753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511371008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877416.445488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877417.213589:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877417.469487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877418.493481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877419.517445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877420.285504:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877420.541481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877420.669452:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877420.669460:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877420.669465:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877420.669476:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2407 00000100:00080000:1.0:1788877420.669487:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877420.669495:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877420.669533:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511371136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877420.669555:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877420.669558:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877420.669559:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877420.669563:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2407 00000100:00080000:1.0:1788877420.669566:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877420.669570:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877420.669579:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877420.669592:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511371264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877420.669600:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877420.669603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877420.669604:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877420.669608:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2407 00000100:00080000:1.0:1788877420.669610:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877420.669613:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877420.669616:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877420.669625:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511371392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877420.669631:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877420.669633:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877420.669635:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877420.669637:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2407 00000100:00080000:1.0:1788877420.669658:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877420.669664:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877420.669668:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877420.669677:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511371520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877420.669684:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877420.669687:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877420.669691:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877420.669696:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2407 00000100:00080000:1.0:1788877420.669702:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877420.669707:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877420.669711:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877420.669720:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511371648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877421.565562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877422.589512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877423.357696:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877423.613512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877424.637461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877425.661496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877425.789511:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877425.789524:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877425.789528:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877425.789540:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2412 00000100:00080000:3.0:1788877425.789552:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877425.789561:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877425.789600:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511371776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877425.789643:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877425.789648:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877425.789650:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877425.789656:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2412 00000100:00080000:3.0:1788877425.789662:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877425.789667:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877425.789678:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877425.789698:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511371904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877425.789720:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877425.789724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877425.789727:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877425.789732:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2412 00000100:00080000:3.0:1788877425.789736:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877425.789742:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877425.789748:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877425.789762:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511372032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877425.789787:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877425.789791:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877425.789794:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877425.789802:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2412 00000100:00080000:3.0:1788877425.789806:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877425.789812:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877425.789818:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877425.789843:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511372160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877425.789903:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877425.789915:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877425.789923:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877425.789942:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2412 00000100:00080000:3.0:1788877425.789948:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877425.789953:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877425.789960:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877425.789988:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511372288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1788877426.429644:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877426.685482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877427.709468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877428.733457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877429.501714:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877429.757513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877430.781467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788877430.909656:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511372416:192.168.203.111@tcp:250:kworker.0 00000800:00000100:3.0:1788877430.909716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877430.909750:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877430/real 1788877430] req@ffffa06198afce00 x1875771511372416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877430.909772:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afce00 x1875771511372416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877430.909791:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877430.909794:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877430.909798:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511372416:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877430.909816:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877430.909831:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877430.909839:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877430/real 1788877430] req@ffffa06198afea00 x1875771511372544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877430.909846:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511372544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877430.909852:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877430.909854:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877430.909857:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877430.909861:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372672:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877430.909872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877430.909877:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877430/real 1788877430] req@ffffa06198afc000 x1875771511372672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877430.909884:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511372672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877430.909890:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877430.909892:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877430.909893:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372672:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877430.909897:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877430.909906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877430.909912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877430/real 1788877430] req@ffffa06198afc700 x1875771511372800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877430.909920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511372800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877430.909924:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877430.909926:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877430.909928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877430.909931:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372928:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877430.909940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877430.909945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877430/real 1788877430] req@ffffa06198afdc00 x1875771511372928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877430.909953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511372928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877430.909957:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877430.909959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877430.909961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511372928:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877431.805461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877432.573569:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877432.829522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877433.853494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877434.877539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877435.645496:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877435.901465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788877436.029606:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511373056:192.168.203.111@tcp:250:kworker.0 00000800:00000100:3.0:1788877436.029686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877436.029733:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877436/real 1788877436] req@ffffa06198aff480 x1875771511373056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877436.029753:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511373056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877436.029764:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877436.029768:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877436.029772:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511373056:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877436.029793:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877436.029813:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877436.029821:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877436/real 1788877436] req@ffffa06198affb80 x1875771511373184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877436.029829:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511373184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877436.029839:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877436.029841:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877436.029844:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877436.029847:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373312:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877436.029858:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877436.029864:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877436/real 1788877436] req@ffffa06198afd880 x1875771511373312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877436.029871:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511373312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877436.029876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877436.029878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877436.029880:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373312:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877436.029884:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877436.029894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877436.029899:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877436/real 1788877436] req@ffffa06198afdf80 x1875771511373440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877436.029906:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511373440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877436.029911:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877436.029913:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877436.029915:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877436.029919:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373568:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877436.029929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877436.029934:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877436/real 1788877436] req@ffffa06198aff800 x1875771511373568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877436.029942:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511373568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877436.029947:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877436.029949:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877436.029951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373568:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877436.925473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877437.949510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877438.717656:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877438.973511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877439.997478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877441.021494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877441.149535:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877441.149546:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877441.149563:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877441.149575:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2427 00000100:00080000:2.0:1788877441.149585:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877441.149594:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877441.149631:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511373696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877441.149662:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877441.149669:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877441.149671:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877441.149677:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2427 00000100:00080000:2.0:1788877441.149682:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877441.149689:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877441.149699:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877441.149715:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511373696:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877441.149718:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511373824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877441.149739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877441.149744:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877441.149746:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877441.149754:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2427 00000100:00080000:2.0:1788877441.149758:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877441.149764:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877441.149770:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877441.149787:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511373952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877441.149798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877441.149803:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877441.149805:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877441.149809:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2427 00000100:00080000:2.0:1788877441.149814:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877441.149819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877441.149827:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877441.149844:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511374080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877441.149857:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877441.149861:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877441.149863:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788877441.149868:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373824:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788877441.149874:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2427 00000100:00080000:2.0:1788877441.149886:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877441.149891:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877441.149898:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877441.149913:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511374208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877441.149927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373952:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877441.149956:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877441.149982:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374208:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877441.789685:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877442.045468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877443.069531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877444.093583:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877444.221889:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877444.221984:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877444.222041:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877444.222132:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877444.222177:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877444.222210:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877444.222240:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788877444.222242:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877441/real 1788877444] req@ffffa061888ad500 x1875771511373824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877444.222303:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511373824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877444.222343:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877444.222355:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877444.222364:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877444.222380:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877441/real 1788877444] req@ffffa061888adf80 x1875771511373952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877444.222461:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511373952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877444.222480:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877444.222488:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877444.222495:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511373952:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877444.222523:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877441/real 1788877444] req@ffffa061888ac000 x1875771511374080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877444.222576:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511374080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877444.222593:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877444.222600:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877444.222606:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877444.222621:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877441/real 1788877444] req@ffffa061888af480 x1875771511373696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877444.222646:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511373696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877444.222663:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877444.222670:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877444.222678:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511373696:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877444.222732:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877441/real 1788877444] req@ffffa061888ae300 x1875771511374208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877444.222757:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511374208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877444.222772:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877444.222779:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877444.222785:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374208:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877444.861675:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877445.117592:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877446.141554:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877446.269618:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877446.269628:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877446.269632:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877446.269652:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2432 00000100:00080000:2.0:1788877446.269663:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877446.269671:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877446.269718:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511374336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877446.269778:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877446.269783:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877446.269788:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877446.269799:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2432 00000100:00080000:2.0:1788877446.269810:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877446.269840:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877446.269852:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511374336:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788877446.269863:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877446.269917:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511374464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877446.269949:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877446.269969:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877446.269971:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877446.269975:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2432 00000100:00080000:2.0:1788877446.269983:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877446.269988:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877446.269995:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788877446.270011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877446.270018:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511374592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877446.270035:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877446.270038:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877446.270040:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877446.270045:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2432 00000100:00080000:2.0:1788877446.270050:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877446.270054:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877446.270061:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877446.270073:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511374720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877446.270086:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877446/real 1788877446] req@ffffa061888ae680 x1875771511374336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877446.270090:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877446.270108:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877446.270115:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877446.270142:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2432 00000100:00080000:2.0:1788877446.270147:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788877446.270151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511374336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877446.270160:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877446.270181:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788877446.270192:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788877446.270208:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511374848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877446.270209:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877446.270228:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511374336:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877446.270293:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877446.270350:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877446.270376:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877446/real 1788877446] req@ffffa061888afb80 x1875771511374464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877446.270441:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511374464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877446.270459:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877446.270931:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877446.270941:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877446.270956:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374592:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877446.271005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877446.271039:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877446/real 1788877446] req@ffffa061888af100 x1875771511374592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877446.271060:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511374592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877446.271070:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877446.271074:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877446.271078:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374592:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877446.271086:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877446.271116:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877446.271136:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877446/real 1788877446] req@ffffa061888adc00 x1875771511374720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877446.271170:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511374720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877446.271185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877446.271190:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877446.271196:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877446.271223:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374848:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877446.271252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877446.271266:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877446/real 1788877446] req@ffffa061888aca80 x1875771511374848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877446.271289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511374848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877446.271303:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877446.271309:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877446.271315:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511374848:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877447.165519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877447.933726:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877448.189508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877449.213466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877450.237526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877451.005536:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877451.261469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877451.389533:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877451.389545:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877451.389550:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877451.389561:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2438 00000100:00080000:1.0:1788877451.389573:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877451.389582:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877451.389618:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511374976/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877451.389647:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877451.389650:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877451.389652:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877451.389657:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2438 00000100:00080000:1.0:1788877451.389662:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877451.389667:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877451.389678:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877451.389693:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511375104/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877451.389707:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877451.389716:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877451.389718:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877451.389722:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2438 00000100:00080000:1.0:1788877451.389725:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877451.389728:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877451.389733:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877451.389746:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511375232/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877451.389756:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877451.389760:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877451.389762:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877451.389767:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2438 00000100:00080000:1.0:1788877451.389771:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877451.389775:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877451.389782:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877451.389796:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511375360/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877451.389807:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877451.389811:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877451.389814:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877451.389818:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2438 00000100:00080000:1.0:1788877451.389825:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877451.389833:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877451.389843:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877451.389860:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511375488/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877452.285551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877453.309447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877454.077768:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877454.333510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877455.357503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877456.381536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877456.509513:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877456.509553:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877456.509559:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877456.509574:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2443 00000100:00080000:1.0:1788877456.509605:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877456.509614:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877456.509667:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511375616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877456.509700:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877456.509705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877456.509708:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877456.509714:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2443 00000100:00080000:1.0:1788877456.509719:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877456.509725:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877456.509737:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877456.509758:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511375744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877456.509772:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877456.509777:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877456.509779:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877456.509784:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2443 00000100:00080000:1.0:1788877456.509805:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877456.509811:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877456.509821:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877456.509838:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511375872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877456.509850:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877456.509854:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877456.509858:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877456.509868:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2443 00000100:00080000:1.0:1788877456.509877:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877456.509882:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877456.509892:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877456.509910:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511376000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877456.509930:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877456.509934:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877456.509936:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877456.509941:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2443 00000100:00080000:1.0:1788877456.509946:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877456.509953:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877456.509964:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877456.509979:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511376128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1788877457.149576:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877457.405516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877458.429529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877459.453519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877460.221600:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877460.477521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877461.501537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877461.629556:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877461.629567:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877461.629574:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877461.629589:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2448 00000100:00080000:3.0:1788877461.629605:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877461.629633:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877461.629682:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511376256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877461.629718:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877461.629723:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877461.629725:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877461.629730:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2448 00000100:00080000:3.0:1788877461.629734:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877461.629739:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877461.629751:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877461.629784:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511376384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877461.629804:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877461.629825:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877461.629829:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877461.629834:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2448 00000100:00080000:3.0:1788877461.629837:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877461.629841:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877461.629847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877461.629872:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511376512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877461.629886:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877461.629892:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877461.629895:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877461.629900:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2448 00000100:00080000:3.0:1788877461.629908:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877461.629914:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877461.629924:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877461.629961:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511376640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877461.629970:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877461.629973:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877461.629976:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877461.629981:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2448 00000100:00080000:3.0:1788877461.629985:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877461.629990:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877461.629995:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877461.630009:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511376768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877462.525481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877463.293659:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877463.549538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877464.573505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877465.597488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877466.621503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788877466.749679:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511376896:192.168.203.111@tcp:250:kworker.0 00000800:00000100:3.0:1788877466.749743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877466.749782:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877466/real 1788877466] req@ffffa06198afed80 x1875771511376896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877466.749822:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511376896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877466.749830:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877466.749834:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877466.749837:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511376896:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877466.749856:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877466.749871:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877466.749879:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877466/real 1788877466] req@ffffa06198aff800 x1875771511377024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877466.749887:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511377024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877466.749893:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877466.749895:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877466.749897:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877466.749901:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377152:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877466.749915:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877466.749926:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877466/real 1788877466] req@ffffa06198afdf80 x1875771511377152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877466.749939:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511377152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877466.749947:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877466.749950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877466.749954:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377152:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877466.749961:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877466.749979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877466.749986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877466/real 1788877466] req@ffffa06198afd880 x1875771511377280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877466.749995:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511377280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877466.750000:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877466.750002:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877466.750004:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877466.750008:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377408:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877466.750017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877466.750023:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877466/real 1788877466] req@ffffa06198affb80 x1875771511377408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877466.750038:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511377408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877466.750043:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877466.750045:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877466.750047:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377408:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877467.645467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877468.669479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877469.693482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877470.717486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877471.741538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788877471.869680:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511377536:192.168.203.111@tcp:250:kworker.0 00000800:00000100:3.0:1788877471.869779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877471.869845:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877471/real 1788877471] req@ffffa06198aff480 x1875771511377536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877471.869873:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511377536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877471.869896:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877471.869902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877471.869928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511377536:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877471.869937:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877471.869963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877471.869979:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877471/real 1788877471] req@ffffa06198afdc00 x1875771511377664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877471.869993:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511377664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877471.870002:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877471.870006:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877471.870010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877471.870035:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377792:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877471.870058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877471.870071:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877471/real 1788877471] req@ffffa06198afc700 x1875771511377792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877471.870085:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511377792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877471.870094:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877471.870098:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877471.870102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377792:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877471.870109:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877471.870133:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877471.870146:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877471/real 1788877471] req@ffffa06198afc000 x1875771511377920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877471.870160:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511377920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877471.870169:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877471.870174:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877471.870177:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511377920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877471.870184:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378048:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877471.870207:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877471.870220:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877471/real 1788877471] req@ffffa06198afea00 x1875771511378048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877471.870234:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511378048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877471.870243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877471.870247:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877471.870250:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378048:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877472.765564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877473.789467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877474.813472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877475.837506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877476.861451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788877476.989643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511378176:192.168.203.111@tcp:250:kworker.0 00000800:00000100:3.0:1788877476.989700:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877476.989737:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877476/real 1788877476] req@ffffa06198afce00 x1875771511378176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877476.989769:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afce00 x1875771511378176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877476.989785:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877476.989792:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877476.989797:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511378176:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877476.989825:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877476.989851:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877476.989863:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877476/real 1788877476] req@ffffa06198afe300 x1875771511378304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877476.989873:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511378304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877476.989882:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877476.989886:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877476.989890:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877476.989897:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378432:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877476.989916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877476.989929:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877476/real 1788877476] req@ffffa06198afca80 x1875771511378432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877476.989941:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511378432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877476.989952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877476.989954:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877476.989957:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378432:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877476.989961:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877476.990001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877476.990012:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877476/real 1788877476] req@ffffa06198afd500 x1875771511378560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877476.990025:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511378560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877476.990034:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877476.990036:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877476.990038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877476.990052:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378688:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877476.990066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877476.990073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877476/real 1788877476] req@ffffa06198afd180 x1875771511378688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877476.990081:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511378688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877476.990086:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877476.990088:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877476.990091:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378688:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877477.885474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877478.909503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877479.933524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877480.957466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877481.981482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877482.109558:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877482.109568:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877482.109572:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877482.109582:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2468 00000100:00080000:1.0:1788877482.109593:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877482.109617:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877482.109652:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511378816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877482.109679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877482.109683:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877482.109696:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877482.109700:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2468 00000100:00080000:1.0:1788877482.109703:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877482.109706:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877482.109732:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877482.109746:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511378944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877482.109756:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877482.109762:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788877482.109762:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511378816:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877482.109764:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877482.109775:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2468 00000100:00080000:1.0:1788877482.109780:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877482.109801:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877482.109809:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877482.109830:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511379072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877482.109845:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788877482.109853:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877482.109856:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877482.109861:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877482.109868:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2468 00000100:00080000:1.0:1788877482.109875:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877482.109880:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877482.109886:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877482.109894:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877482/real 1788877482] req@ffffa06188699180 x1875771511378816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877482.109903:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511379200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877482.109925:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877482.109929:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788877482.109931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511378816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877482.109933:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877482.109944:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2468 00000100:00080000:1.0:1788877482.109953:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877482.109957:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877482.109959:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877482.109967:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788877482.109968:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877482.109977:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511378816:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877482.109984:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511379328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877482.110033:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877482.110056:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877482.110067:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877482/real 1788877482] req@ffffa0618869b800 x1875771511378944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877482.110078:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511378944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877482.110085:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877482.110088:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877482.110091:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511378944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877482.110095:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379072:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877482.110108:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877482.110116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877482/real 1788877482] req@ffffa0618869aa00 x1875771511379072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877482.110125:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511379072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877482.110131:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877482.110133:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877482.110136:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379072:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877482.110140:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877482.110152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877482.110159:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877482/real 1788877482] req@ffffa0618869b100 x1875771511379200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877482.110168:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511379200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877482.110175:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877482.110178:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877482.110181:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877482.110189:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379328:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877482.110211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877482.110226:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877482/real 1788877482] req@ffffa06188698380 x1875771511379328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877482.110242:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511379328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877482.110252:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877482.110255:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877482.110257:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379328:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877483.005488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877484.029459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877485.053474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877486.077497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877487.101508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877487.229561:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877487.229568:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877487.229574:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877487.229584:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2473 00000100:00080000:1.0:1788877487.229594:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877487.229602:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877487.229636:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511379456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877487.229656:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877487.229659:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877487.229661:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877487.229664:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2473 00000100:00080000:1.0:1788877487.229667:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877487.229674:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877487.229685:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511379456:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788877487.229686:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877487.229703:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511379584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877487.229717:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877487.229720:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877487.229723:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877487.229729:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2473 00000100:00080000:1.0:1788877487.229733:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877487.229736:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877487.229740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1788877487.229753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877487.229754:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511379712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877487.229763:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877487.229767:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877487.229769:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877487.229773:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2473 00000100:00080000:1.0:1788877487.229777:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877487.229781:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877487.229786:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877487.229797:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877487/real 1788877487] req@ffffa06188699c00 x1875771511379456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877487.229798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511379840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877487.229824:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877487.229828:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877487.229830:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877487.229833:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2473 00000100:00080000:1.0:1788877487.229836:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877487.229837:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511379456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877487.229841:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877487.229847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1788877487.229852:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788877487.229857:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511379968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877487.229859:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877487.229867:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511379456:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877487.229890:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877487.229907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877487.229943:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877487/real 1788877487] req@ffffa06188698380 x1875771511379584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877487.229967:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511379584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877487.229981:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877487.229985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877487.229987:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877487.229992:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379712:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877487.230013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877487.230022:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877487/real 1788877487] req@ffffa0618869b100 x1875771511379712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877487.230032:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511379712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877487.230040:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877487.230043:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877487.230047:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379712:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877487.230054:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877487.230076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877487.230090:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877487/real 1788877487] req@ffffa0618869aa00 x1875771511379840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877487.230101:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511379840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877487.230113:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877487.230116:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877487.230118:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877487.230123:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379968:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877487.230135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877487.230142:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877487/real 1788877487] req@ffffa0618869b800 x1875771511379968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877487.230151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511379968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877487.230158:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877487.230160:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877487.230163:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511379968:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877488.125515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877489.149526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877490.173554:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877491.197488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877492.221488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877492.349566:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877492.349579:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877492.349585:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877492.349600:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2479 00000100:00080000:1.0:1788877492.349618:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877492.349627:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877492.349692:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511380096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877492.349732:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877492.349735:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877492.349737:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877492.349742:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2479 00000100:00080000:1.0:1788877492.349746:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877492.349752:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877492.349764:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877492.349778:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511380224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877492.349787:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877492.349790:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877492.349792:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877492.349794:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2479 00000100:00080000:1.0:1788877492.349796:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877492.349800:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877492.349803:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877492.349812:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511380352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877492.349821:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877492.349825:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877492.349826:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877492.349830:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2479 00000100:00080000:1.0:1788877492.349833:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877492.349838:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877492.349842:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877492.349853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511380096:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877492.349854:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511380480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877492.349884:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877492.349888:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877492.349889:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877492.349895:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2479 00000100:00080000:1.0:1788877492.349899:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877492.349902:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877492.349907:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877492.349921:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511380608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877492.349956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877492.350026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877492/real 1788877492] req@ffffa06188699180 x1875771511380096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877492.350055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511380096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877492.350073:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877492.350079:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877492.350094:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511380096:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877492.350109:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877492.350147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877492.350168:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877492/real 1788877492] req@ffffa06188699f80 x1875771511380224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877492.350183:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511380224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877492.350192:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877492.350195:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877492.350198:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877492.350214:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380352:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877492.350236:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877492.350246:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877492/real 1788877492] req@ffffa06188698000 x1875771511380352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877492.350257:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511380352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877492.350264:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877492.350267:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877492.350269:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380352:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877492.350303:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877492.350354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877492.350374:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877492/real 1788877492] req@ffffa0618869ad80 x1875771511380480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877492.350442:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511380480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877492.350451:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877492.350454:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877492.350457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877492.350464:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380608:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877492.350488:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877492.350500:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877492/real 1788877492] req@ffffa0618869bb80 x1875771511380608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877492.350511:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511380608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877492.350519:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877492.350522:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877492.350525:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380608:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877493.245510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877494.269460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877495.293520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877496.317451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877497.341473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877497.469571:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877497.469587:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877497.469595:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877497.469610:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2484 00000100:00080000:3.0:1788877497.469626:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877497.469638:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877497.469693:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511380736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877497.469754:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877497.469758:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877497.469761:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877497.469765:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2484 00000100:00080000:3.0:1788877497.469770:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877497.469775:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877497.469796:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877497.469827:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511380736:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877497.469839:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511380864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877497.469864:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877497.469872:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877497.469877:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877497.469885:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2484 00000100:00080000:3.0:1788877497.469907:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877497.469916:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877497.469927:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1788877497.469934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877497.469968:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511380992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877497.469982:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877497.469986:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877497.469988:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877497.469994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877497/real 1788877497] req@ffffa06188872680 x1875771511380736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877497.469995:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2484 00000100:00080000:3.0:1788877497.470003:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877497.470013:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877497.470021:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877497.470061:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511380736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877497.470073:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511381120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877497.470145:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877497.470159:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877497.470161:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788877497.470170:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877497.470177:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877497.470180:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511380736:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877497.470187:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2484 00000100:00080000:3.0:1788877497.470213:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877497.470218:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877497.470223:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877497.470236:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877497.470240:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511381248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877497.470276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877497.470290:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877497/real 1788877497] req@ffffa06188870700 x1875771511380864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877497.470321:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511380864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877497.470333:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877497.470336:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877497.470338:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877497.470343:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380992:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877497.470361:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877497.470369:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877497/real 1788877497] req@ffffa06188873b80 x1875771511380992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877497.470378:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511380992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877497.470412:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877497.470414:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877497.470417:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511380992:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877497.470422:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877497.470437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877497.470446:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877497/real 1788877497] req@ffffa06188872300 x1875771511381120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877497.470454:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511381120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877497.470460:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877497.470462:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877497.470465:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877497.470477:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381248:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877497.470489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877497.470495:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877497/real 1788877497] req@ffffa06188873480 x1875771511381248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877497.470502:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511381248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877497.470508:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877497.470510:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877497.470512:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381248:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877498.365478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877499.389485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877500.413465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877501.437512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877502.461494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877502.589547:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877502.589579:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877502.589585:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877502.589599:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2489 00000100:00080000:3.0:1788877502.589613:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877502.589622:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877502.589666:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511381376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877502.589693:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877502.589696:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877502.589698:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877502.589701:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2489 00000100:00080000:3.0:1788877502.589704:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877502.589707:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877502.589716:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877502.589726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511381504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877502.589735:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877502.589738:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877502.589740:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877502.589749:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2489 00000100:00100000:2.0:1788877502.589751:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511381376:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877502.589752:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877502.589763:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877502.589768:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877502.589780:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511381632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877502.589787:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877502.589790:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877502.589793:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877502.589796:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2489 00000100:00080000:3.0:1788877502.589798:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877502.589801:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877502.589806:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877502.589814:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511381760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877502.589820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877502.589823:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877502.589828:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788877502.589829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877502.589839:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2489 00000100:00080000:3.0:1788877502.589842:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877502.589845:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877502.589847:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877502.589857:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511381888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877502.589886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877502/real 1788877502] req@ffffa06188871f80 x1875771511381376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877502.589902:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511381376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877502.589911:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877502.589914:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877502.589917:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511381376:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877502.589937:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877502.589960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877502.589969:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877502/real 1788877502] req@ffffa06188872a00 x1875771511381504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877502.589977:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511381504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877502.589983:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877502.589985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877502.589987:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877502.589991:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381632:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877502.590002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877502.590008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877502/real 1788877502] req@ffffa06188873100 x1875771511381632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877502.590015:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511381632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877502.590025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877502.590027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877502.590030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381632:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877502.590034:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877502.590044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877502.590050:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877502/real 1788877502] req@ffffa06188871500 x1875771511381760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877502.590057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511381760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877502.590062:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877502.590064:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877502.590066:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877502.590070:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381888:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877502.590079:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877502.590085:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877502/real 1788877502] req@ffffa06188873800 x1875771511381888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877502.590092:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511381888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877502.590097:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877502.590099:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877502.590102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511381888:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877503.485573:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877504.509609:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877505.533558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877506.557514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877507.581492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877507.709474:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877507.709482:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877507.709486:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877507.709495:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2494 00000100:00080000:3.0:1788877507.709504:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877507.709511:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877507.709545:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511382016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877507.709565:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877507.709568:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877507.709569:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877507.709572:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2494 00000100:00080000:3.0:1788877507.709574:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877507.709577:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877507.709585:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877507.709594:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511382144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877507.709602:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511382016:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877507.709605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877507.709611:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877507.709612:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877507.709615:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2494 00000100:00080000:3.0:1788877507.709617:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877507.709619:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877507.709623:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877507.709632:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511382272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877507.709639:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877507.709641:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877507.709643:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877507.709646:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2494 00000100:00080000:3.0:1788877507.709651:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877507.709654:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877507.709657:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877507.709664:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511382400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877507.709670:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877507.709673:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877507.709674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877507.709677:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2494 00000100:00080000:3.0:1788877507.709679:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877507.709682:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877507.709685:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877507.709691:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511382528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877507.709697:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877507.709723:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382272:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877507.709740:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877507.709757:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382528:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877508.605516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877509.629510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877510.653535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788877510.781735:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788877510.781774:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788877510.781787:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877510.781825:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877510.781836:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877510.781845:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877510.781856:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788877510.781880:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877507/real 1788877510] req@ffffa06188871f80 x1875771511382528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877510.781911:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511382528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877510.781925:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877510.781930:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877510.781934:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382528:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877510.781950:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877507/real 1788877510] req@ffffa06188871500 x1875771511382144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877510.781964:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511382144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877510.781972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877510.781975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877510.781978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877510.781986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877507/real 1788877510] req@ffffa06188873100 x1875771511382272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877510.781998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511382272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877510.782006:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877510.782009:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877510.782012:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382272:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877510.782018:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877507/real 1788877510] req@ffffa06188872a00 x1875771511382400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877510.782029:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511382400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877510.782036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877510.782038:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877510.782041:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877510.782047:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877507/real 1788877510] req@ffffa06188873800 x1875771511382016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877510.782058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511382016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877510.782065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877510.782068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877510.782071:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511382016:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877511.677460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877512.701511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877512.829498:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877512.829506:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877512.829509:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877512.829518:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2499 00000100:00080000:1.0:1788877512.829528:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877512.829535:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877512.829567:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511382656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877512.829588:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877512.829591:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877512.829592:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877512.829595:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2499 00000100:00080000:1.0:1788877512.829597:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877512.829600:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877512.829607:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877512.829616:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511382784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877512.829623:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877512.829626:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877512.829628:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877512.829630:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2499 00000100:00080000:1.0:1788877512.829632:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877512.829634:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877512.829637:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877512.829646:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511382912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877512.829646:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511382656:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877512.829661:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877512.829663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877512.829664:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877512.829668:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2499 00000100:00080000:1.0:1788877512.829672:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877512.829675:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877512.829680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877512.829693:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511383040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877512.829703:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877512.829707:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877512.829709:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877512.829713:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2499 00000100:00080000:1.0:1788877512.829717:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788877512.829723:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877512.829727:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877512.829734:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877512.829748:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511383168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877512.829764:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877512/real 1788877512] req@ffffa0618869a680 x1875771511382656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877512.829792:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511382656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877512.829802:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877512.829807:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877512.829810:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511382656:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877512.829818:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877512.829845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877512.829858:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877512/real 1788877512] req@ffffa0618869bb80 x1875771511382784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877512.829871:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511382784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877512.829880:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877512.829884:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877512.829888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877512.829913:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382912:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877512.829934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877512.829946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877512/real 1788877512] req@ffffa0618869ad80 x1875771511382912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877512.829959:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511382912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877512.829967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877512.829970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877512.829974:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511382912:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877512.829979:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877512.829999:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877512.830010:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877512/real 1788877512] req@ffffa06188698000 x1875771511383040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877512.830022:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511383040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877512.830030:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877512.830033:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877512.830037:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877512.830043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383168:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877512.830061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877512.830073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877512/real 1788877512] req@ffffa06188699f80 x1875771511383168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877512.830084:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511383168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877512.830098:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877512.830101:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877512.830105:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383168:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877513.725475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877514.749512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877515.773481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877516.797502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877517.821459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877517.950097:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511383296:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877517.950149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877517.950177:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877517/real 1788877517] req@ffffa06198afc380 x1875771511383296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877517.950192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511383296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877517.950201:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877517.950216:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877517.950221:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511383296:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877517.950227:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877517.950248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877517.950261:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877517/real 1788877517] req@ffffa06198afd180 x1875771511383424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877517.950272:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511383424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877517.950279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877517.950281:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877517.950284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877517.950291:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383552:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877517.950311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877517.950322:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877517/real 1788877517] req@ffffa06198afd500 x1875771511383552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877517.950332:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511383552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877517.950339:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877517.950343:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877517.950345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383552:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877517.950351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877517.950370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877517.950380:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877517/real 1788877517] req@ffffa06198afca80 x1875771511383680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877517.950416:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511383680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877517.950425:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877517.950427:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877517.950430:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877517.950435:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383808:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877517.950455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877517.950466:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877517/real 1788877517] req@ffffa06198afe300 x1875771511383808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877517.950477:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511383808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877517.950484:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877517.950487:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877517.950490:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511383808:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877518.845478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877519.869461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877520.893472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877521.917491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877522.941528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877523.069848:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877523.069858:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877523.069862:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877523.069872:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2509 00000100:00080000:3.0:1788877523.069882:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877523.069890:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877523.069927:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511383936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877523.069966:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877523.069970:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877523.069972:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877523.069977:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2509 00000100:00080000:3.0:1788877523.069980:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877523.069985:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877523.069995:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877523.070008:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511384064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877523.070018:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877523.070021:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877523.070023:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877523.070026:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2509 00000100:00080000:3.0:1788877523.070030:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877523.070034:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877523.070039:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877523.070049:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511384192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877523.070058:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877523.070061:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877523.070062:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877523.070065:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2509 00000100:00080000:3.0:1788877523.070068:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877523.070072:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877523.070076:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877523.070084:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511384320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877523.070092:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877523.070095:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877523.070097:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877523.070100:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2509 00000100:00080000:3.0:1788877523.070103:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877523.070106:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877523.070111:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877523.070121:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511384448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877523.070237:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511383936:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877523.070313:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877523.070346:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877523/real 1788877523] req@ffffa06188873480 x1875771511383936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877523.070364:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511383936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877523.070374:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877523.070378:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877523.070412:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511383936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877523.070420:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877523.070450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877523.070465:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877523/real 1788877523] req@ffffa06188872300 x1875771511384064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877523.070478:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511384064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877523.070492:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877523.070496:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877523.070499:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877523.070504:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384192:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877523.070529:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877523.070542:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877523/real 1788877523] req@ffffa06188873b80 x1875771511384192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877523.070556:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511384192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877523.070564:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877523.070567:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877523.070570:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384192:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877523.070576:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877523.070594:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877523.070605:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877523/real 1788877523] req@ffffa06188870700 x1875771511384320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877523.070616:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511384320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877523.070624:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877523.070626:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877523.070630:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877523.070635:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384448:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877523.070650:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877523.070660:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877523/real 1788877523] req@ffffa06188872680 x1875771511384448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877523.070670:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511384448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877523.070687:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877523.070693:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877523.070696:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384448:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877523.965489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877524.989550:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877526.013456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877527.037453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877528.061500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877528.189480:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877528.189490:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877528.189495:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877528.189504:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2514 00000100:00080000:3.0:1788877528.189514:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877528.189521:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877528.189553:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511384576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877528.189631:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511384576:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877528.189685:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877528.189714:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877528.189717:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877528.189719:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877528.189724:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2514 00000100:00080000:3.0:1788877528.189728:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877528.189732:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877528.189740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877528.189753:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511384704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877528.190045:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877528/real 1788877528] req@ffffa06188871c00 x1875771511384576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877528.190062:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511384576/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877528.190072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877528.190076:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877528.190079:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511384576:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877528.190100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877528.190121:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877528.190132:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877528/real 1788877528] req@ffffa06188872d80 x1875771511384704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877528.190143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511384704/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877528.190151:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877528.190153:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877528.190172:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384704:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788877528.190250:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877528.190254:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877528.190255:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877528.190259:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2514 00000100:00080000:3.0:1788877528.190263:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877528.190268:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877528.190273:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877528.190285:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511384832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877528.190323:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384832:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877528.190344:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877528.190363:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877528.190367:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877528.190369:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877528.190373:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2514 00000100:00080000:3.0:1788877528.190377:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877528.190397:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877528.190402:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877528.190413:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511384960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877528.190480:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877528/real 1788877528] req@ffffa06188870380 x1875771511384832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877528.190492:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511384832/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877528.190500:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877528.190503:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877528.190506:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384832:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877528.190517:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877528.190536:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877528.190544:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877528/real 1788877528] req@ffffa06188870a80 x1875771511384960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877528.190555:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511384960/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877528.190563:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877528.190566:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877528.190569:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511384960:192.168.203.111@tcp:8:kworker.0 00000100:00080000:3.0:1788877528.191002:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877528.191006:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877528.191008:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877528.191012:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2514 00000100:00080000:3.0:1788877528.191016:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877528.191019:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877528.191025:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877528.191037:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511385088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877528.191075:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385088:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877528.191096:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877528.191150:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877528/real 1788877528] req@ffffa06188871880 x1875771511385088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877528.191173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511385088/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877528.191181:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877528.191184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877528.191187:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385088:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877529.085472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877530.109498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877531.133520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877532.157498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877533.181452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877533.309593:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511385216:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877533.309653:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877533.309687:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877533/real 1788877533] req@ffffa06198afce00 x1875771511385216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877533.309705:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afce00 x1875771511385216/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877533.309714:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877533.309719:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877533.309725:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511385216:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877533.309743:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877533.309763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877533.309776:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877533/real 1788877533] req@ffffa06198afea00 x1875771511385344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877533.309793:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511385344/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877533.309800:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877533.309803:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877533.309806:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877533.309810:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385472:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877533.309822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877533.309828:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877533/real 1788877533] req@ffffa06198afc000 x1875771511385472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877533.309834:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511385472/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877533.309838:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877533.309840:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877533.309842:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385472:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877533.309845:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877533.309853:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877533.309858:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877533/real 1788877533] req@ffffa06198afc700 x1875771511385600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877533.309864:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511385600/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877533.309869:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877533.309870:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877533.309872:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877533.309875:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385728:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877533.309883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877533.309888:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877533/real 1788877533] req@ffffa06198afdc00 x1875771511385728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877533.309893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511385728/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877533.309898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877533.309899:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877533.309901:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385728:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877534.205492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877535.229466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877536.253496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877537.277468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877538.301485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877538.429541:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877538.429556:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877538.429562:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877538.429577:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2525 00000100:00080000:3.0:1788877538.429592:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877538.429603:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877538.429659:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511385856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877538.429697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877538.429702:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877538.429705:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877538.429710:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2525 00000100:00080000:3.0:1788877538.429715:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877538.429721:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877538.429733:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877538.429744:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511385856:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877538.429752:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511385984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877538.429766:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877538.429771:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877538.429773:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877538.429777:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2525 00000100:00080000:3.0:1788877538.429781:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877538.429786:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877538.429792:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877538.429806:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511386112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877538.429819:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877538.429830:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877538.429835:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877538.429838:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877538.429843:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2525 00000100:00080000:3.0:1788877538.429848:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877538.429853:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877538.429860:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877538.429862:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877538/real 1788877538] req@ffffa06188871880 x1875771511385856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877538.429877:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511386240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877538.429882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511385856/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877538.429907:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877538.429913:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877538.429913:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877538.429917:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877538.429919:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877538.429928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511385856:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877538.429928:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2525 00000100:00080000:3.0:1788877538.429937:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877538.429945:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877538.429951:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877538.429956:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877538.429967:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511386368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877538.429978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877538.429996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877538/real 1788877538] req@ffffa06188870a80 x1875771511385984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877538.430011:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511385984/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877538.430021:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877538.430025:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877538.430030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511385984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877538.430037:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386112:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877538.430059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877538.430071:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877538/real 1788877538] req@ffffa06188870380 x1875771511386112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877538.430085:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511386112/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877538.430094:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877538.430098:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877538.430101:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386112:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877538.430106:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877538.430139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877538.430147:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877538/real 1788877538] req@ffffa06188872d80 x1875771511386240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877538.430156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511386240/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877538.430161:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877538.430164:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877538.430166:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877538.430179:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386368:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877538.430190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877538.430197:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877538/real 1788877538] req@ffffa06188871c00 x1875771511386368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877538.430204:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511386368/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877538.430209:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877538.430212:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877538.430214:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386368:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877539.325499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877540.349501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877541.373476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877542.397531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877543.421444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877543.549442:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877543.549451:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877543.549455:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877543.549463:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2530 00000100:00080000:3.0:1788877543.549473:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877543.549485:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877543.549525:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511386496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877543.549552:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877543.549556:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877543.549557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877543.549562:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2530 00000100:00080000:3.0:1788877543.549565:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877543.549568:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877543.549577:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877543.549593:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511386624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877543.549606:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877543.549612:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1788877543.549615:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511386496:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877543.549615:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877543.549624:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2530 00000100:00080000:3.0:1788877543.549629:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877543.549632:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877543.549636:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877543.549648:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511386752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877543.549655:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877543.549658:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877543.549660:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877543.549662:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2530 00000100:00080000:3.0:1788877543.549673:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877543.549676:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877543.549680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877543.549691:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511386880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877543.549692:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877543.549701:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877543.549704:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877543.549705:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877543.549708:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2530 00000100:00080000:3.0:1788877543.549711:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877543.549714:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877543.549716:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877543.549723:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511387008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877543.549740:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877543/real 1788877543] req@ffffa06188872680 x1875771511386496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877543.549762:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511386496/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877543.549774:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877543.549778:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877543.549782:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511386496:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877543.549806:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877543.549829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877543.549843:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877543/real 1788877543] req@ffffa06188870700 x1875771511386624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877543.549855:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511386624/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877543.549865:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877543.549868:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877543.549872:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877543.549878:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386752:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877543.549899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877543.549911:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877543/real 1788877543] req@ffffa06188873b80 x1875771511386752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877543.549923:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511386752/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877543.549938:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877543.549940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877543.549944:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386752:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877543.549950:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877543.549982:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877543.549995:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877543/real 1788877543] req@ffffa06188872300 x1875771511386880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877543.550006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511386880/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877543.550014:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877543.550017:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877543.550020:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511386880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877543.550026:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387008:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877543.550043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877543.550053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877543/real 1788877543] req@ffffa06188873480 x1875771511387008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877543.550064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511387008/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877543.550071:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877543.550073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877543.550077:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387008:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877544.445469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877545.469511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877546.493478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877547.517484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877548.541495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877548.669505:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877548.669515:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877548.669521:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877548.669531:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2535 00000100:00080000:1.0:1788877548.669541:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877548.669549:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877548.669599:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511387136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877548.669630:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877548.669634:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877548.669637:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877548.669641:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2535 00000100:00080000:1.0:1788877548.669646:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877548.669651:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877548.669661:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877548.669672:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511387136:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877548.669678:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511387264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877548.669693:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877548.669698:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877548.669700:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877548.669705:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2535 00000100:00080000:1.0:1788877548.669711:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877548.669715:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877548.669721:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877548.669735:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511387392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877548.669740:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877548.669758:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877548.669763:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877548.669766:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877548.669771:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2535 00000100:00080000:1.0:1788877548.669776:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877548.669781:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877548.669786:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877548/real 1788877548] req@ffffa06188699180 x1875771511387136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788877548.669789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877548.669815:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511387520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877548.669815:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511387136/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877548.669837:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877548.669842:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877548.669843:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788877548.669851:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1788877548.669851:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511387136:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877548.669856:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877548.669862:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2535 00000100:00080000:1.0:1788877548.669867:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877548.669871:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877548.669876:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387264:192.168.203.111@tcp:38:kworker.0 00000008:00100000:1.0:1788877548.669877:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877548.669891:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511387648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877548.669908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877548.669923:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877548/real 1788877548] req@ffffa0618869b800 x1875771511387264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877548.669937:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511387264/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877548.669946:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877548.669950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877548.669954:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877548.669961:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387392:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877548.669984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877548.669997:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877548/real 1788877548] req@ffffa0618869aa00 x1875771511387392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877548.670011:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511387392/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877548.670020:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877548.670023:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877548.670027:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387392:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877548.670031:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877548.670048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877548.670056:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877548/real 1788877548] req@ffffa0618869b100 x1875771511387520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877548.670064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511387520/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877548.670085:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877548.670087:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877548.670089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877548.670101:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387648:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877548.670116:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877548.670124:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877548/real 1788877548] req@ffffa06188698380 x1875771511387648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877548.670131:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511387648/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877548.670137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877548.670139:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877548.670141:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387648:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877549.565477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877550.589453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877551.613489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877552.637467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877553.661504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877553.789462:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877553.789471:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877553.789475:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877553.789483:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2540 00000100:00080000:1.0:1788877553.789495:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877553.789504:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877553.789542:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511387776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877553.789566:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877553.789569:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877553.789570:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877553.789573:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2540 00000100:00080000:1.0:1788877553.789575:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877553.789578:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877553.789591:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877553.789603:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511387904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877553.789610:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511387776:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877553.789614:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877553.789618:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877553.789620:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877553.789624:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2540 00000100:00080000:1.0:1788877553.789626:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877553.789629:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877553.789633:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877553.789642:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511388032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877553.789651:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877553.789656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877553.789657:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877553.789661:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2540 00000100:00080000:1.0:1788877553.789665:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788877553.789670:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877553.789674:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877553.789681:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877553.789693:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511388160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877553.789702:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877553.789705:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877553/real 1788877553] req@ffffa06188699c00 x1875771511387776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877553.789707:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877553.789712:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877553.789721:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2540 00000100:00080000:1.0:1788877553.789725:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877553.789729:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511387776/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877553.789731:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877553.789740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1788877553.789744:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877553.789751:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877553.789755:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511388288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877553.789760:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511387776:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877553.789794:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877553.789817:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877553.789827:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877553/real 1788877553] req@ffffa06188698380 x1875771511387904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877553.789835:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511387904/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877553.789842:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877553.789844:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877553.789846:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511387904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877553.789851:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388032:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877553.789862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877553.789868:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877553/real 1788877553] req@ffffa0618869b100 x1875771511388032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877553.789876:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511388032/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877553.789881:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877553.789882:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877553.789884:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388032:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877553.789887:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877553.789897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877553.789903:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877553/real 1788877553] req@ffffa0618869aa00 x1875771511388160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877553.789910:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511388160/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877553.789915:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877553.789917:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877553.789918:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877553.789922:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388288:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877553.789931:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877553.789936:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877553/real 1788877553] req@ffffa0618869b800 x1875771511388288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877553.789944:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511388288/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877553.789949:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877553.789950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877553.789952:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388288:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877554.685512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877555.709536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877556.733445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877557.757479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877558.525585:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877558.781439:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877558.909506:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877558.909519:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877558.909525:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877558.909540:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2545 00000100:00080000:3.0:1788877558.909555:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877558.909564:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877558.909611:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511388416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877558.909641:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877558.909644:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877558.909646:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877558.909650:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2545 00000100:00080000:3.0:1788877558.909652:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877558.909655:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877558.909666:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877558.909676:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511388544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877558.909684:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877558.909687:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877558.909688:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877558.909692:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2545 00000100:00080000:3.0:1788877558.909696:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877558.909698:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511388416:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877558.909700:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877558.909706:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877558.909715:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511388672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877558.909721:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877558.909723:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877558.909725:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877558.909727:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2545 00000100:00080000:3.0:1788877558.909731:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877558.909734:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877558.909738:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877558.909746:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511388800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877558.909752:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877558.909755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877558.909756:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877558.909759:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2545 00000100:00080000:3.0:1788877558.909770:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877558.909773:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877558.909775:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877558.909785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877558.909786:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511388928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877558.909828:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877558/real 1788877558] req@ffffa06188873800 x1875771511388416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877558.909845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511388416/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877558.909855:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877558.909858:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877558.909862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511388416:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877558.909885:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877558.909906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877558.909920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877558/real 1788877558] req@ffffa06188872a00 x1875771511388544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877558.909934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511388544/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877558.909951:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877558.909954:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877558.909958:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877558.909965:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388672:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877558.909988:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877558.910001:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877558/real 1788877558] req@ffffa06188873100 x1875771511388672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877558.910010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511388672/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877558.910016:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877558.910031:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877558.910035:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388672:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877558.910040:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877558.910065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877558.910073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877558/real 1788877558] req@ffffa06188871500 x1875771511388800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877558.910082:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511388800/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877558.910087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877558.910090:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877558.910092:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877558.910096:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388928:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877558.910108:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877558.910114:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877558/real 1788877558] req@ffffa06188871f80 x1875771511388928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877558.910121:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511388928/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877558.910126:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877558.910127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877558.910129:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511388928:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877559.805514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877560.829517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877561.597582:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877561.853520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877562.877497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877563.901535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877564.029572:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877564.029609:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877564.029617:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877564.029634:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2550 00000100:00080000:3.0:1788877564.029650:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877564.029663:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877564.029720:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511389056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877564.029759:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877564.029767:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877564.029770:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877564.029776:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2550 00000100:00080000:3.0:1788877564.029781:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877564.029786:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877564.029798:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877564.029816:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511389184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877564.029833:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877564.029840:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511389056:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877564.029840:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877564.029844:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877564.029856:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2550 00000100:00080000:3.0:1788877564.029861:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877564.029865:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877564.029871:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877564.029889:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511389312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877564.029902:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877564.029906:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877564.029909:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877564.029913:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2550 00000100:00080000:3.0:1788877564.029918:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877564.029923:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877564.029930:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877564.029938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877564.029946:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511389440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877564.029961:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877564.029966:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877564.029969:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877564.029973:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2550 00000100:00080000:3.0:1788877564.029979:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877564.029984:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877564.029990:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877564.030028:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877564/real 1788877564] req@ffffa06188871f80 x1875771511389056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877564.030036:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511389568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877564.030087:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511389056/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877564.030112:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877564.030119:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877564.030124:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511389056:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877564.030157:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877564.030192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877564.030212:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877564/real 1788877564] req@ffffa06188871500 x1875771511389184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877564.030237:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511389184/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877564.030254:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877564.030261:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877564.030267:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877564.030280:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389312:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877564.030320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877564.030339:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877564/real 1788877564] req@ffffa06188873100 x1875771511389312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877564.030355:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511389312/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877564.030366:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877564.030370:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877564.030374:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389312:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877564.030381:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877564.030450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877564.030465:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877564/real 1788877564] req@ffffa06188872a00 x1875771511389440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877564.030480:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511389440/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877564.030491:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877564.030495:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877564.030498:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877564.030505:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389568:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877564.030524:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877564.030536:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877564/real 1788877564] req@ffffa06188873800 x1875771511389568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877564.030556:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511389568/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877564.030571:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877564.030577:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877564.030583:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389568:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877564.669641:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877564.925517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877565.949508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877566.973543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877567.741608:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877567.997463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877569.021432:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877569.149451:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877569.149459:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877569.149462:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877569.149472:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2555 00000100:00080000:3.0:1788877569.149481:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877569.149488:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877569.149519:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511389696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877569.149553:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877569.149556:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877569.149557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877569.149560:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2555 00000100:00080000:3.0:1788877569.149563:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877569.149566:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877569.149573:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877569.149582:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511389824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877569.149593:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877569.149596:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877569.149597:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877569.149599:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2555 00000100:00080000:3.0:1788877569.149601:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877569.149608:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877569.149611:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877569.149617:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511389952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877569.149624:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877569.149627:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877569.149629:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877569.149631:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2555 00000100:00080000:3.0:1788877569.149633:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877569.149636:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877569.149639:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877569.149648:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511390080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877569.149649:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511389696:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877569.149667:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877569.149669:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877569.149671:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877569.149673:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2555 00000100:00080000:3.0:1788877569.149675:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877569.149677:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877569.149680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877569.149688:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511390208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877569.149714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877569.149744:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877569/real 1788877569] req@ffffa06188873480 x1875771511389696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877569.149758:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511389696/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877569.149766:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877569.149768:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877569.149771:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511389696:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877569.149776:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877569.149790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877569.149797:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877569/real 1788877569] req@ffffa06188872300 x1875771511389824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877569.149805:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511389824/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877569.149810:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877569.149812:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877569.149814:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877569.149817:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389952:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877569.149827:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877569.149833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877569/real 1788877569] req@ffffa06188873b80 x1875771511389952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877569.149840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511389952/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877569.149845:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877569.149847:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877569.149849:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511389952:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877569.149867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877569.149878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877569.149886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877569/real 1788877569] req@ffffa06188870700 x1875771511390080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877569.149908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511390080/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877569.149916:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877569.149920:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877569.149924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877569.149930:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390208:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877569.149951:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877569.149962:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877569/real 1788877569] req@ffffa06188872680 x1875771511390208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877569.149970:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511390208/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877569.149991:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877569.149993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877569.149995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390208:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877570.045474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877570.813629:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877571.069477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877572.093501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877572.861629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877573.117474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877573.885718:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877574.141507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877574.269879:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511390336:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877574.270058:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877574.270123:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390592:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877574.270183:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877574.270233:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390848:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877575.165480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877575.933595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877576.189452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877576.957659:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877577.213481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877577.341626:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877577.341665:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877577.341678:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877577.341710:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877577.341716:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877577.341722:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877577.341734:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788877577.341774:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877574/real 1788877577] req@ffffa06198afdf80 x1875771511390720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877577.341814:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511390720/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877577.341828:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877577.341834:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877577.341839:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877577.341849:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877574/real 1788877577] req@ffffa06198aff800 x1875771511390848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877577.341859:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511390848/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877577.341869:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877577.341874:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877577.341877:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390848:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877577.341885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877574/real 1788877577] req@ffffa06198affb80 x1875771511390464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877577.341899:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511390464/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877577.341908:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877577.341912:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877577.341916:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877577.341924:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877574/real 1788877577] req@ffffa06198afd880 x1875771511390592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877577.341955:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511390592/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877577.341963:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877577.341967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877577.341970:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511390592:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877577.341979:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877574/real 1788877577] req@ffffa06198aff480 x1875771511390336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877577.341994:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511390336/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877577.342003:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877577.342006:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877577.342010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511390336:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877578.237443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877579.005643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877579.261474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877579.389672:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511390976:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877579.389816:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877579.389923:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877579/real 1788877579] req@ffffa06198afed80 x1875771511390976/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877579.390001:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511390976/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877579.390016:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877579.390024:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877579.390029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511390976:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877579.390071:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877579.390108:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877579.390127:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877579/real 1788877579] req@ffffa06198aff480 x1875771511391104/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877579.390144:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511391104/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877579.390156:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877579.390160:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877579.390164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877579.390173:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391232:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877579.390201:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877579.390217:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877579/real 1788877579] req@ffffa06198afd880 x1875771511391232/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877579.390232:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511391232/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877579.390242:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877579.390246:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877579.390250:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391232:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877579.390274:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877579.390299:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877579.390312:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877579/real 1788877579] req@ffffa06198affb80 x1875771511391360/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877579.390326:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511391360/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877579.390336:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877579.390339:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877579.390343:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877579.390351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391488:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877579.390376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877579.390390:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877579/real 1788877579] req@ffffa06198aff800 x1875771511391488/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877579.390403:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511391488/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877579.390455:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877579.390461:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877579.390465:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391488:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877580.285496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877581.309489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877582.077574:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877582.333489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877583.357512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877584.381502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877584.509591:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877584.509614:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877584.509619:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877584.509630:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2571 00000100:00080000:1.0:1788877584.509643:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877584.509655:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877584.509714:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511391616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877584.509747:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877584.509753:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877584.509765:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877584.509770:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2571 00000100:00080000:1.0:1788877584.509776:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877584.509782:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877584.509795:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877584.509814:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511391744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877584.509831:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877584.509835:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877584.509837:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877584.509841:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2571 00000100:00080000:1.0:1788877584.509847:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877584.509850:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877584.509856:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877584.509888:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511391872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877584.509888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511391616:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877584.509978:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877584.509984:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877584.509987:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877584.509996:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2571 00000100:00080000:1.0:1788877584.510003:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877584.510009:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877584.510019:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877584.510057:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511392000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877584.510072:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877584.510075:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877584.510080:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877584.510083:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877584.510106:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2571 00000100:00080000:1.0:1788877584.510114:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877584.510119:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877584.510128:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877584.510138:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877584/real 1788877584] req@ffffa06188699180 x1875771511391616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877584.510146:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511392128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877584.510233:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511391616/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877584.510246:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877584.510251:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877584.510256:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511391616:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877584.510265:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877584.510286:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877584.510296:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877584/real 1788877584] req@ffffa06188699f80 x1875771511391744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877584.510308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511391744/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877584.510319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877584.510323:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877584.510327:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877584.510364:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391872:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877584.510388:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877584.510443:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877584/real 1788877584] req@ffffa06188698000 x1875771511391872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877584.510454:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511391872/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877584.510467:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877584.510470:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877584.510472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511391872:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877584.510479:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877584.510496:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877584.510505:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877584/real 1788877584] req@ffffa0618869ad80 x1875771511392000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877584.510514:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511392000/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877584.510520:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877584.510523:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877584.510525:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877584.510530:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392128:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877584.510542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877584.510549:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877584/real 1788877584] req@ffffa0618869bb80 x1875771511392128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877584.510558:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511392128/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877584.510563:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877584.510565:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877584.510569:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392128:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877585.149519:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877585.405466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877586.429488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877587.453461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877588.221566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877588.477479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877589.501444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877589.629501:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877589.629510:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877589.629514:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877589.629523:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2576 00000100:00080000:3.0:1788877589.629532:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877589.629540:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877589.629573:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511392256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877589.629595:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877589.629598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877589.629599:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877589.629602:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2576 00000100:00080000:3.0:1788877589.629604:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877589.629607:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877589.629615:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877589.629627:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511392384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877589.629636:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877589.629638:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877589.629641:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877589.629643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511392256:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877589.629646:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2576 00000100:00080000:3.0:1788877589.629651:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877589.629654:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877589.629658:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877589.629666:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511392512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877589.629673:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877589.629676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877589.629677:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877589.629679:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2576 00000100:00080000:3.0:1788877589.629692:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877589.629695:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877589.629698:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877589.629706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877589.629709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511392640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877589.629716:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877589.629719:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877589.629721:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877589.629723:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2576 00000100:00080000:3.0:1788877589.629726:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877589.629729:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877589.629731:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877589.629738:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511392768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877589.629748:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877589/real 1788877589] req@ffffa06188871c00 x1875771511392256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877589.629771:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511392256/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877589.629780:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877589.629783:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877589.629786:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511392256:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877589.629805:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877589.629824:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877589.629832:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877589/real 1788877589] req@ffffa06188872d80 x1875771511392384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877589.629840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511392384/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877589.629846:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877589.629848:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877589.629851:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877589.629854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392512:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877589.629864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877589.629871:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877589/real 1788877589] req@ffffa06188870380 x1875771511392512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877589.629878:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511392512/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877589.629899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877589.629902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877589.629905:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392512:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877589.629909:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877589.629924:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877589.629932:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877589/real 1788877589] req@ffffa06188870a80 x1875771511392640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877589.629940:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511392640/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877589.629945:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877589.629947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877589.629949:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877589.629953:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392768:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877589.629962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877589.629968:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877589/real 1788877589] req@ffffa06188871880 x1875771511392768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877589.629974:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511392768/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877589.629980:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877589.629982:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877589.629984:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511392768:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877590.525523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877591.293590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877591.549488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877592.573456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877593.597462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877594.365518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877594.621513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788877594.749666:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511392896:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877594.749742:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877594.749793:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877594/real 1788877594] req@ffffa06198afdf80 x1875771511392896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877594.749880:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511392896/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877594.749898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877594.749905:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877594.749912:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511392896:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877594.749945:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877594.749979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877594.749998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877594/real 1788877594] req@ffffa06198afdc00 x1875771511393024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877594.750016:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511393024/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877594.750028:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877594.750034:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877594.750038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877594.750048:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393152:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877594.750076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877594.750092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877594/real 1788877594] req@ffffa06198afc700 x1875771511393152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877594.750110:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511393152/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877594.750122:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877594.750127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877594.750131:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393152:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877594.750140:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877594.750166:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877594.750181:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877594/real 1788877594] req@ffffa06198afc000 x1875771511393280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877594.750199:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511393280/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877594.750210:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877594.750215:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877594.750220:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877594.750227:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393408:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877594.750251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877594.750280:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877594/real 1788877594] req@ffffa06198afea00 x1875771511393408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877594.750298:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511393408/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877594.750311:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877594.750315:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877594.750320:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393408:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877595.645450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788877596.669473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877597.437708:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877599.869635:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877599.869648:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877599.869654:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877599.869668:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2586 00000100:00080000:3.0:1788877599.869682:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877599.869693:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877599.869749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511393536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877599.869770:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877599.869773:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877599.869774:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877599.869777:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2586 00000100:00080000:3.0:1788877599.869779:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877599.869782:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877599.869789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877599.869798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511393664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877599.869809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511393536:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877599.869815:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877599.869819:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877599.869820:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877599.869823:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2586 00000100:00080000:3.0:1788877599.869826:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877599.869828:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877599.869831:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877599.869855:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511393792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877599.869862:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877599.869865:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877599.869867:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877599.869869:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2586 00000100:00080000:3.0:1788877599.869873:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788877599.869878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877599.869879:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877599.869895:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877599.869903:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511393920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877599.869910:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877599.869913:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877599.869914:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877599.869917:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2586 00000100:00080000:3.0:1788877599.869920:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877599.869923:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877599.869926:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877599.869927:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877599/real 1788877599] req@ffffa06188871880 x1875771511393536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877599.869936:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511394048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877599.869946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511393536/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877599.869956:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877599.869959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877599.869962:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511393536:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877599.869991:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877599.870006:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877599.870014:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877599/real 1788877599] req@ffffa06188870a80 x1875771511393664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877599.870022:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511393664/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877599.870052:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877599.870054:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877599.870057:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877599.870061:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393792:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877599.870073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877599.870080:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877599/real 1788877599] req@ffffa06188870380 x1875771511393792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877599.870087:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511393792/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877599.870093:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877599.870107:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877599.870109:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393792:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877599.870112:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877599.870123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877599.870128:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877599/real 1788877599] req@ffffa06188872d80 x1875771511393920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877599.870136:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511393920/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877599.870141:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877599.870142:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877599.870145:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511393920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877599.870149:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394048:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877599.870158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877599.870164:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877599/real 1788877599] req@ffffa06188871c00 x1875771511394048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877599.870171:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511394048/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877599.870176:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877599.870177:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877599.870179:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394048:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877600.509601:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877601.533708:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877603.581547:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877604.605552:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877604.989525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877604.989536:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877604.989540:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877604.989552:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2591 00000100:00080000:1.0:1788877604.989567:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877604.989576:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877604.989622:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511394176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877604.989647:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877604.989650:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877604.989651:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877604.989654:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2591 00000100:00080000:1.0:1788877604.989657:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877604.989659:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877604.989669:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877604.989686:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511394304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877604.989694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877604.989697:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877604.989699:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877604.989702:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2591 00000100:00080000:1.0:1788877604.989704:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877604.989707:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877604.989711:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877604.989719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511394432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877604.989733:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877604.989733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511394176:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877604.989748:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877604.989750:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877604.989754:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2591 00000100:00080000:1.0:1788877604.989756:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877604.989758:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877604.989761:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877604.989773:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511394560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877604.989780:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877604.989782:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877604.989783:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877604.989786:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2591 00000100:00080000:1.0:1788877604.989788:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877604.989791:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877604.989795:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877604.989802:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511394688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877604.989838:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877604.989873:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877604/real 1788877604] req@ffffa0618869a680 x1875771511394176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877604.989888:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511394176/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877604.989897:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877604.989900:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877604.989903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511394176:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877604.989909:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877604.989926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877604.989934:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877604/real 1788877604] req@ffffa0618869bb80 x1875771511394304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877604.989942:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511394304/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877604.989948:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877604.989950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877604.989953:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877604.989972:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394432:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877604.989982:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877604.989988:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877604/real 1788877604] req@ffffa0618869ad80 x1875771511394432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877604.989996:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511394432/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877604.990001:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877604.990004:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877604.990006:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394432:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877604.990010:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877604.990020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877604.990026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877604/real 1788877604] req@ffffa06188698000 x1875771511394560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877604.990033:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511394560/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877604.990039:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877604.990041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877604.990043:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877604.990046:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394688:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877604.990055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877604.990061:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877604/real 1788877604] req@ffffa06188699f80 x1875771511394688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877604.990067:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511394688/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877604.990072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877604.990074:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877604.990076:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394688:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877606.653543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877607.677702:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877609.725672:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877610.109520:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877610.109534:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877610.109542:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877610.109560:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2596 00000100:00080000:1.0:1788877610.109581:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877610.109596:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877610.109673:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511394816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877610.109707:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877610.109712:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877610.109714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877610.109720:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2596 00000100:00080000:1.0:1788877610.109725:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877610.109729:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877610.109751:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877610.109777:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511394944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877610.109830:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877610.109836:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877610.109839:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877610.109846:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2596 00000100:00100000:2.0:1788877610.109855:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511394816:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877610.109856:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877610.109871:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877610.109882:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877610.109908:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511395072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877610.109941:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1788877610.109941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877610.109951:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877610.109955:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877610.109962:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2596 00000100:00080000:1.0:1788877610.109971:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877610.109977:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877610.109986:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877610.109993:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877610/real 1788877610] req@ffffa06188699180 x1875771511394816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877610.110012:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511395200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877610.110019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511394816/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877610.110059:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877610.110060:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877610.110068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788877610.110072:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1788877610.110078:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511394816:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877610.110080:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877610.110103:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394944:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788877610.110105:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2596 00000100:00080000:1.0:1788877610.110130:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788877610.110131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877610.110146:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877610.110153:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877610/real 1788877610] req@ffffa0618869b800 x1875771511394944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788877610.110160:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877610.110173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511394944/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877610.110185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788877610.110190:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511395328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877610.110194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877610.110207:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511394944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877610.110256:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395072:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877610.110276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877610.110286:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877610/real 1788877610] req@ffffa0618869aa00 x1875771511395072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877610.110297:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511395072/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877610.110305:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877610.110308:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877610.110311:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395072:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877610.110317:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877610.110333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877610.110349:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877610/real 1788877610] req@ffffa0618869b100 x1875771511395200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877610.110361:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511395200/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877610.110369:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877610.110372:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877610.110374:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877610.110417:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395328:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877610.110440:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877610.110452:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877610/real 1788877610] req@ffffa06188698380 x1875771511395328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877610.110464:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511395328/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877610.110472:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877610.110475:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877610.110478:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395328:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877610.749623:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877612.797550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877613.821647:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877615.229534:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877615.229542:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877615.229546:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877615.229555:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2601 00000100:00080000:3.0:1788877615.229564:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877615.229570:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877615.229602:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511395456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877615.229624:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877615.229627:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877615.229628:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877615.229631:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2601 00000100:00080000:3.0:1788877615.229634:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877615.229636:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877615.229643:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877615.229653:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511395584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877615.229661:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877615.229665:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877615.229667:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877615.229672:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2601 00000100:00080000:3.0:1788877615.229674:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877615.229677:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877615.229680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877615.229687:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511395712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877615.229693:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877615.229696:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877615.229698:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877615.229701:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2601 00000100:00080000:3.0:1788877615.229703:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877615.229705:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877615.229708:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877615.229722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511395840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877615.229723:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511395456:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877615.229733:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877615.229744:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877615.229745:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877615.229749:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2601 00000100:00080000:3.0:1788877615.229752:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877615.229754:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877615.229758:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877615.229781:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511395968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877615.229879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877615.229932:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877615/real 1788877615] req@ffffa06188872680 x1875771511395456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877615.229956:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511395456/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877615.229976:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877615.229981:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877615.229988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511395456:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877615.230001:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877615.230039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877615.230061:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877615/real 1788877615] req@ffffa06188870700 x1875771511395584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877615.230079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511395584/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877615.230088:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877615.230092:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877615.230095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877615.230102:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395712:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877615.230125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877615.230136:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877615/real 1788877615] req@ffffa06188873b80 x1875771511395712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877615.230147:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511395712/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877615.230155:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877615.230159:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877615.230162:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395712:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877615.230205:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877615.230239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877615.230258:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877615/real 1788877615] req@ffffa06188872300 x1875771511395840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877615.230272:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511395840/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877615.230281:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877615.230284:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877615.230288:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877615.230294:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395968:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877615.230319:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877615.230336:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877615/real 1788877615] req@ffffa06188873480 x1875771511395968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877615.230356:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511395968/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877615.230370:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877615.230375:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877615.230381:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511395968:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877615.869544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877616.893567:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877618.941666:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877619.965676:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877620.349556:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877620.349567:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877620.349573:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877620.349586:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2607 00000100:00080000:1.0:1788877620.349600:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877620.349608:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877620.349654:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511396096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877620.349683:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877620.349687:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877620.349689:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877620.349692:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2607 00000100:00080000:1.0:1788877620.349696:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877620.349699:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877620.349710:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877620.349726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511396224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877620.349728:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511396096:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877620.349762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877620.349767:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877620.349770:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877620.349775:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2607 00000100:00080000:1.0:1788877620.349806:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877620.349811:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877620.349818:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877620.349830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511396352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877620.349837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877620.349841:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877620.349842:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877620.349845:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2607 00000100:00080000:1.0:1788877620.349848:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877620.349850:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788877620.349855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788877620.349856:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877620.349866:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511396480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877620.349873:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877620.349876:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877620.349878:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877620.349880:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2607 00000100:00080000:1.0:1788877620.349882:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877620.349885:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877620.349899:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877620.349908:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511396608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877620.349915:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877620/real 1788877620] req@ffffa06188699c00 x1875771511396096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877620.349934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511396096/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877620.349943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877620.349946:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877620.349949:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511396096:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877620.349974:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877620.350021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877620.350031:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877620/real 1788877620] req@ffffa06188698380 x1875771511396224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877620.350040:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511396224/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877620.350047:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877620.350065:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877620.350068:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877620.350073:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396352:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877620.350086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877620.350094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877620/real 1788877620] req@ffffa0618869b100 x1875771511396352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877620.350103:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511396352/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877620.350131:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877620.350134:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877620.350136:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396352:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877620.350140:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877620.350152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877620.350158:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877620/real 1788877620] req@ffffa0618869aa00 x1875771511396480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877620.350165:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511396480/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877620.350170:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877620.350172:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877620.350174:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877620.350177:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396608:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877620.350215:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877620.350222:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877620/real 1788877620] req@ffffa0618869b800 x1875771511396608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877620.350230:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511396608/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877620.350236:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877620.350238:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877620.350241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396608:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877622.013574:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877623.037550:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877625.085547:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877625.469557:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877625.469570:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877625.469576:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877625.469590:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2612 00000100:00080000:1.0:1788877625.469604:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877625.469616:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877625.469663:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699180 x1875771511396736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877625.469703:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877625.469710:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877625.469713:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877625.469745:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2612 00000100:00080000:1.0:1788877625.469753:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877625.469764:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877625.469790:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877625.469816:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511396864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877625.469821:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511396736:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877625.469838:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877625.469844:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877625.469857:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877625.469863:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2612 00000100:00080000:1.0:1788877625.469869:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877625.469875:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877625.469886:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877625.469902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511396992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877625.469918:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877625.469924:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877625.469927:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788877625.469927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877625.469934:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2612 00000100:00080000:1.0:1788877625.469939:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877625.469946:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877625.469954:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877625.469971:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511397120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877625.469982:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877625/real 1788877625] req@ffffa06188699180 x1875771511396736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877625.469990:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877625.469997:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877625.470000:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877625.470007:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2612 00000100:00080000:1.0:1788877625.470014:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877625.470014:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699180 x1875771511396736/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877625.470021:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877625.470030:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1788877625.470033:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877625.470041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877625.470047:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511397248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877625.470049:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511396736:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877625.470085:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877625.470130:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877625.470154:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877625/real 1788877625] req@ffffa06188699f80 x1875771511396864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877625.470171:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511396864/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877625.470183:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877625.470187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877625.470191:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877625.470199:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396992:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877625.470224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877625.470237:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877625/real 1788877625] req@ffffa06188698000 x1875771511396992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877625.470251:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511396992/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877625.470261:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877625.470264:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877625.470268:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511396992:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877625.470274:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877625.470294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877625.470305:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877625/real 1788877625] req@ffffa0618869ad80 x1875771511397120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877625.470319:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511397120/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877625.470328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877625.470331:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877625.470335:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877625.470341:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397248:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877625.470359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877625.470369:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877625/real 1788877625] req@ffffa0618869bb80 x1875771511397248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877625.470419:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511397248/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877625.470431:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877625.470434:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877625.470438:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397248:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877626.109551:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877628.157648:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877629.181622:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877630.589571:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877630.589587:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877630.589594:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877630.589609:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2617 00000100:00080000:1.0:1788877630.589624:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877630.589635:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877630.589719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511397376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877630.589786:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877630.589791:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877630.589793:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877630.589799:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2617 00000100:00080000:1.0:1788877630.589803:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877630.589809:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877630.589821:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877630.589862:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511397504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877630.589874:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511397376:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877630.589908:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877630.589915:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877630.589919:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877630.589928:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2617 00000100:00080000:1.0:1788877630.589936:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877630.589944:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877630.589971:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788877630.589990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877630.589998:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511397632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877630.590017:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877630.590023:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877630.590027:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788877630.590037:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877630/real 1788877630] req@ffffa0618869a680 x1875771511397376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877630.590038:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2617 00000100:00080000:1.0:1788877630.590062:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877630.590075:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877630.590076:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511397376/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788877630.590090:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877630.590128:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511397760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877630.590128:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877630.590137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877630.590147:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511397376:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877630.590155:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877630.590161:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877630.590163:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877630.590169:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2617 00000100:00080000:1.0:1788877630.590190:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788877630.590192:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397504:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788877630.590202:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877630.590213:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877630.590228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877630.590239:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511397888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877630.590248:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877630/real 1788877630] req@ffffa0618869bb80 x1875771511397504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877630.590289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511397504/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877630.590299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877630.590303:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877630.590310:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877630.590325:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397632:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877630.590342:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877630.590352:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877630/real 1788877630] req@ffffa0618869ad80 x1875771511397632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877630.590364:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511397632/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877630.590371:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877630.590374:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877630.590377:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397632:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877630.590382:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877630.590398:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877630.590406:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877630/real 1788877630] req@ffffa06188698000 x1875771511397760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877630.590473:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511397760/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877630.590480:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877630.590482:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877630.590485:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877630.590498:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397888:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877630.590542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877630.590554:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877630/real 1788877630] req@ffffa06188699f80 x1875771511397888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877630.590573:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511397888/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877630.590581:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877630.590584:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877630.590587:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511397888:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877631.229774:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877632.253532:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877634.301613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877635.325550:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877635.709501:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877635.709515:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877635.709521:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877635.709535:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2622 00000100:00080000:2.0:1788877635.709570:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877635.709585:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877635.709635:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511398016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877635.709712:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877635.709718:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877635.709721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877635.709728:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2622 00000100:00080000:2.0:1788877635.709733:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877635.709740:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877635.709752:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877635.709772:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511398144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877635.709774:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511398016:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877635.709788:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877635.709792:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877635.709795:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877635.709800:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2622 00000100:00080000:2.0:1788877635.709804:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877635.709809:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877635.709816:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877635.709828:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511398272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877635.709841:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877635.709845:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877635.709848:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877635.709852:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2622 00000100:00080000:2.0:1788877635.709857:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788877635.709859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877635.709862:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877635.709867:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877635.709880:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511398400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877635.709913:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877635.709919:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788877635.709922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877635/real 1788877635] req@ffffa061888ad880 x1875771511398016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877635.709923:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877635.709930:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2622 00000100:00080000:2.0:1788877635.709935:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877635.709942:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877635.709973:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877635.709977:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511398016/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877635.709987:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511398528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877635.709992:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877635.709998:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877635.710003:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511398016:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877635.710055:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877635.710100:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877635.710120:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877635/real 1788877635] req@ffffa061888ac380 x1875771511398144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877635.710136:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511398144/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877635.710156:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877635.710160:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877635.710165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877635.710173:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398272:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877635.710196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877635.710208:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877635/real 1788877635] req@ffffa061888ad180 x1875771511398272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877635.710222:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511398272/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877635.710232:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877635.710235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877635.710239:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398272:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877635.710245:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877635.710264:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877635.710291:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877635/real 1788877635] req@ffffa061888ac700 x1875771511398400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877635.710301:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511398400/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877635.710308:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877635.710311:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877635.710314:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877635.710319:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398528:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877635.710334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877635.710341:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877635/real 1788877635] req@ffffa061888aed80 x1875771511398528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877635.710351:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511398528/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877635.710358:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877635.710360:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877635.710363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398528:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877637.373662:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877638.397592:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877640.445635:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877640.829506:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877640.829516:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877640.829521:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877640.829533:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2627 00000100:00080000:2.0:1788877640.829563:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877640.829574:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877640.829621:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511398656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877640.829697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877640.829702:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877640.829704:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877640.829709:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2627 00000100:00080000:2.0:1788877640.829713:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877640.829717:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877640.829727:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877640.829742:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511398784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877640.829754:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877640.829761:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788877640.829762:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511398656:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877640.829763:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877640.829775:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2627 00000100:00080000:2.0:1788877640.829779:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877640.829783:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877640.829787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877640.829802:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511398912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877640.829811:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877640.829814:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877640.829816:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877640.829820:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2627 00000100:00080000:2.0:1788877640.829824:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877640.829828:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877640.829832:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877640.829843:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511399040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877640.829852:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877640.829855:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877640.829857:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877640.829860:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2627 00000100:00080000:2.0:1788877640.829865:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877640.829869:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877640.829874:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877640.829883:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511399168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877640.829888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877640.829936:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398912:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877640.829965:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877640.829991:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399168:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877641.469697:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877643.517712:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788877643.901695:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877643.901739:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877643.901756:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877643.901791:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877643.901797:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877643.901802:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877643.901812:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788877643.901870:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877640/real 1788877643] req@ffffa061888ad180 x1875771511399168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877643.901908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511399168/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877643.901930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877643.901935:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877643.901941:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399168:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877643.901951:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877640/real 1788877643] req@ffffa061888ace00 x1875771511398784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877643.901965:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511398784/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877643.901974:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877643.901978:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877643.901982:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877643.901990:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877640/real 1788877643] req@ffffa061888aed80 x1875771511398912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877643.902004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511398912/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877643.902012:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877643.902016:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877643.902020:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511398912:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877643.902028:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877640/real 1788877643] req@ffffa061888ac700 x1875771511399040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877643.902041:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511399040/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877643.902050:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877643.902054:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877643.902057:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877643.902066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877640/real 1788877643] req@ffffa061888aea00 x1875771511398656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877643.902079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511398656/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877643.902088:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877643.902092:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877643.902095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511398656:192.168.203.111@tcp:250:kworker.0 00010000:00010000:1.0:1788877644.541576:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877645.949458:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877645.949466:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877645.949470:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877645.949479:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2632 00000100:00080000:2.0:1788877645.949489:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877645.949496:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877645.949528:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511399296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877645.949549:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877645.949551:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877645.949553:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877645.949556:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2632 00000100:00080000:2.0:1788877645.949558:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877645.949560:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877645.949568:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877645.949576:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511399424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877645.949583:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877645.949585:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877645.949587:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877645.949589:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2632 00000100:00080000:2.0:1788877645.949591:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877645.949594:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877645.949597:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877645.949603:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511399552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877645.949611:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877645.949616:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788877645.949616:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511399296:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877645.949618:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877645.949641:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2632 00000100:00080000:2.0:1788877645.949645:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877645.949649:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877645.949655:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877645.949671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511399680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877645.949681:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877645.949685:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877645.949687:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877645.949693:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2632 00000800:00000100:3.0:1788877645.949698:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877645.949700:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877645.949708:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877645.949713:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877645.949725:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511399808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877645.949731:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877645/real 1788877645] req@ffffa061888ac380 x1875771511399296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877645.949757:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511399296/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877645.949767:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877645.949772:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877645.949777:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511399296:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877645.949784:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877645.949817:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877645.949831:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877645/real 1788877645] req@ffffa061888ad880 x1875771511399424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877645.949844:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511399424/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877645.949863:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877645.949866:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877645.949868:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877645.949896:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399552:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877645.949920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877645.949933:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877645/real 1788877645] req@ffffa061888aca80 x1875771511399552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877645.949945:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511399552/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877645.949954:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877645.949958:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877645.949961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399552:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877645.949968:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877645.949989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877645.950001:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877645/real 1788877645] req@ffffa061888adc00 x1875771511399680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877645.950019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511399680/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877645.950027:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877645.950030:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877645.950033:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877645.950039:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399808:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877645.950060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877645.950087:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877645/real 1788877645] req@ffffa061888af100 x1875771511399808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877645.950100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511399808/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877645.950109:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877645.950112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877645.950115:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511399808:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877646.589594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877647.613510:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877649.661650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877650.685594:0:15192:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877651.069565:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877651.069583:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877651.069594:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877651.069641:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2637 00000100:00080000:2.0:1788877651.069665:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877651.069682:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877651.069752:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511399936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877651.069796:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877651.069801:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877651.069804:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877651.069810:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2637 00000100:00080000:2.0:1788877651.069816:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877651.069821:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877651.069835:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877651.069852:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511400064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877651.069867:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877651.069872:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877651.069875:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877651.069880:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2637 00000100:00080000:2.0:1788877651.069884:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877651.069889:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877651.069896:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877651.069900:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511399936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877651.069913:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511400192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877651.069928:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877651.069932:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877651.069934:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877651.069938:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2637 00000100:00080000:2.0:1788877651.069941:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877651.069945:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877651.069951:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877651.069986:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511400320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877651.069995:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877651.069998:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877651.070000:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877651.070004:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2637 00000100:00080000:2.0:1788877651.070020:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877651.070025:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877651.070031:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877651.070042:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877651.070047:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511400448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877651.070099:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877651/real 1788877651] req@ffffa061888afb80 x1875771511399936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877651.070122:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511399936/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877651.070142:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877651.070146:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877651.070151:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511399936:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877651.070159:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877651.070182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877651.070194:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877651/real 1788877651] req@ffffa061888ae680 x1875771511400064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877651.070206:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511400064/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877651.070214:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877651.070217:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877651.070221:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877651.070252:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400192:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877651.070269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877651.070279:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877651/real 1788877651] req@ffffa061888ae300 x1875771511400192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877651.070290:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511400192/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877651.070298:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877651.070301:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877651.070304:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400192:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877651.070311:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877651.070328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877651.070337:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877651/real 1788877651] req@ffffa061888af480 x1875771511400320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877651.070348:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511400320/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877651.070356:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877651.070358:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877651.070361:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877651.070366:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400448:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877651.070381:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877651.070389:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877651/real 1788877651] req@ffffa061888ac000 x1875771511400448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877651.070436:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511400448/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877651.070445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877651.070448:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877651.070451:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400448:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877652.733651:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877653.757585:0:43317:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 5/5 locks to regular blp list, 1 blwis in pool 00010000:00010000:1.0:1788877653.757638:0:15192:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 5 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1788877653.757651:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887ba00/0x7d14dcdc37048ae0 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xf5bf0f7e313b0acc expref: -99 pid: 42950 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00010000:1.0:1788877653.757692:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa06180031e00/0x7d14dcdc37048aee lrc: 2/0,0 mode: PR/PR res: [0x200001b71:0x115:0x0].0x0 bits 0x1b/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313b0afd expref: -99 pid: 42951 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000080:00200000:1.0:1788877653.757712:0:15192:0:(vvp_object.c:117:vvp_conf_set()) [0x200001b71:0x115:0x0]: losing layout lock 00010000:00010000:1.0:1788877653.757745:0:15192:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffffa06189393000 lock: ffffa0618887b400/0x7d14dcdc37048ae7 lrc: 2/0,0 mode: CR/CR res: [0x200001b71:0x118:0x0].0x0 bits 0xd/0x0 rrc: 3 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xf5bf0f7e313b0ada expref: -99 pid: 42950 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000002:00100000:1.0:1788877653.757819:0:15192:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffffa06188871180 x1875771511326592/t94489280524(94489280524) o101->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 576/664 e 0 to 0 dl 1788877072 ref 1 fl Complete:RPQU/604/ffffffff rc 0/-1 job:'grep.0' uid:0 gid:0 projid:0 00000020:00100000:1.0:1788877653.757832:0:15192:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffffa06189393000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:1.0:1788877653.757838:0:15192:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffffa06188699180 x1875771511400704/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00080000:1.0:1788877653.757861:0:15192:0:(client.c:1846:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffffa06188699180 x1875771511400704/t0(0) o35->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:0 00000100:00100000:1.0:1788877653.757870:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877654.781471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877655.805462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:3.0:1788877656.189541:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877656.189551:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877656.189556:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877656.189566:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2642 00000100:00080000:3.0:1788877656.189578:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877656.189599:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877656.189645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511400832/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877656.189674:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877656.189677:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877656.189679:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877656.189682:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2642 00000100:00080000:3.0:1788877656.189685:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877656.189689:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877656.189699:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877656.189709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511400960/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877656.189718:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877656.189724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877656.189725:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877656.189726:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511400832:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877656.189730:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2642 00000100:00080000:3.0:1788877656.189740:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877656.189745:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877656.189749:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877656.189760:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511401088/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877656.189773:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877656.189781:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877656.189783:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877656.189787:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2642 00000100:00080000:3.0:1788877656.189792:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877656.189795:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877656.189800:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877656.189806:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877656.189811:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511401216/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877656.189822:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877656.189826:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877656.189827:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877656.189834:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2642 00000100:00080000:3.0:1788877656.189839:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877656.189842:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877656.189846:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877656.189850:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877656/real 1788877656] req@ffffa06188873800 x1875771511400832/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877656.189861:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871f80 x1875771511401344/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877656.189883:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511400832/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877656.189901:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877656.189904:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877656.189908:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511400832:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877656.189927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400960:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877656.189944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877656.189951:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877656/real 1788877656] req@ffffa06188872a00 x1875771511400960/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877656.189959:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511400960/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877656.189965:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877656.189967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877656.189969:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511400960:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877656.189973:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401088:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877656.189983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877656.189990:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877656/real 1788877656] req@ffffa06188873100 x1875771511401088/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877656.189998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511401088/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877656.190007:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877656.190010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877656.190013:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401088:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877656.190020:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401216:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877656.190042:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877656.190054:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877656/real 1788877656] req@ffffa06188871500 x1875771511401216/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877656.190064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511401216/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877656.190070:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877656.190072:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877656.190074:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401216:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877656.190078:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401344:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877656.190092:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877656.190099:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877656/real 1788877656] req@ffffa06188871f80 x1875771511401344/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877656.190106:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871f80 x1875771511401344/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877656.190111:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877656.190113:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877656.190115:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401344:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877656.829535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877657.853510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877658.877506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877659.901552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877660.925545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:1.0:1788877661.309602:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877661.309614:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877661.309620:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877661.309635:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2648 00000100:00080000:1.0:1788877661.309650:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877661.309662:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877661.309722:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511401472/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877661.309762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877661.309766:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877661.309770:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877661.309775:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2648 00000100:00080000:1.0:1788877661.309780:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877661.309786:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877661.309801:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877661.309819:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511401600/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877661.309835:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877661.309842:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877661.309846:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877661.309854:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2648 00000100:00100000:2.0:1788877661.309854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511401472:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877661.309862:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877661.309866:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877661.309876:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877661.309894:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511401728/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877661.309909:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877661.309914:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877661.309918:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877661.309923:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2648 00000100:00080000:1.0:1788877661.309927:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877661.309932:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877661.309939:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877661.309956:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511401856/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877661.309969:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877661.309976:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877661.309982:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877661.309989:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2648 00000100:00080000:1.0:1788877661.309997:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877661.310009:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788877661.310011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788877661.310033:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877661.310075:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511401984/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877661.310122:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877661/real 1788877661] req@ffffa0618869b800 x1875771511401472/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877661.310150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511401472/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877661.310166:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877661.310172:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877661.310178:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511401472:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877661.310215:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401600:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877661.310272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877661.310291:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877661/real 1788877661] req@ffffa0618869aa00 x1875771511401600/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877661.310308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511401600/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877661.310321:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877661.310325:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877661.310330:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401600:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877661.310338:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401728:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877661.310360:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877661.310374:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877661/real 1788877661] req@ffffa0618869b100 x1875771511401728/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877661.310398:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511401728/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877661.310482:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877661.310486:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877661.310489:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401728:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877661.310496:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401856:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877661.310546:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877661.310604:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877661/real 1788877661] req@ffffa06188698380 x1875771511401856/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877661.310615:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511401856/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877661.310622:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877661.310625:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877661.310627:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401856:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877661.310632:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401984:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877661.310654:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877661.310668:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877661/real 1788877661] req@ffffa06188699c00 x1875771511401984/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877661.310684:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511401984/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877661.310695:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877661.310699:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877661.310704:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511401984:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877661.949488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788877661.949521:0:43317:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1788877661.949542:0:43317:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00010000:1.0:1788877661.949581:0:15191:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00080000:1.0:1788877661.949805:0:2396:0:(client.c:1846:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffffa06198afce00 x1875771511402112/t0(0) o103->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1788877661.949880:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877661.949883:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877661.949885:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877661.949909:0:15191:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 2/0,0 mode: PW/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800028400000000 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000080:00200000:3.0:1788877661.949956:0:15191:0:(vvp_io.c:1826:vvp_io_init()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00000080:00200000:3.0:1788877661.949982:0:15191:0:(vvp_io.c:304:vvp_io_fini()) [0x200001b71:0x118:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 1 00010000:00010000:3.0:1788877661.950007:0:15191:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 1/0,0 mode: --/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00010000:00010000:3.0:1788877661.950015:0:15191:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1788877661.950040:0:15191:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffffa06189393000 lock: 00000000d0fa4fc5/0x7d14dcdc37048ad9 lrc: 0/0,0 mode: --/PW res: [0x280000401:0x302:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x804869400000020 nid: local remote: 0xf5bf0f7e313b0abe expref: -99 pid: 42718 timeout: 0 lvb_type: 1 lru_score: 5 lru_type: 0 00000100:00080000:2.0:1788877661.950057:0:2398:0:(client.c:1846:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffffa06188871f80 x1875771511402240/t0(0) o103->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1788877661.950060:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877661.950065:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877661.950069:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877662.973527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877663.997590:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877663.997670:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877665.021468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788877665.021603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877666.045644:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:2.0:1788877666.429592:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877666.429600:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877666.429603:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877666.429611:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2653 00000100:00080000:2.0:1788877666.429620:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877666.429625:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877666.429658:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511402368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877666.429682:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877666.429685:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877666.429686:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877666.429689:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2653 00000100:00080000:2.0:1788877666.429692:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877666.429694:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877666.429701:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877666.429710:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511402496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877666.429722:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877666.429724:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877666.429726:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877666.429728:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2653 00000100:00080000:2.0:1788877666.429730:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877666.429735:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877666.429735:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511402368:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788877666.429741:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877666.429750:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511402624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877666.429757:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877666.429760:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877666.429761:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877666.429763:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2653 00000100:00080000:2.0:1788877666.429766:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877666.429768:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877666.429771:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877666.429779:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511402752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877666.429785:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877666.429787:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877666.429788:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877666.429790:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2653 00000100:00080000:2.0:1788877666.429793:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788877666.429795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877666.429799:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877666.429803:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877666.429812:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511402880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877666.429833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877666/real 1788877666] req@ffffa061888adf80 x1875771511402368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877666.429847:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511402368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877666.429855:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877666.429858:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877666.429861:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511402368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877666.429866:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877666.429882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877666.429890:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877666/real 1788877666] req@ffffa061888ad500 x1875771511402496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877666.429898:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511402496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877666.429903:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877666.429905:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877666.429907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877666.429925:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877666.429935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877666.429940:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877666/real 1788877666] req@ffffa061888ac000 x1875771511402624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877666.429948:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511402624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877666.429953:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877666.429954:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877666.429956:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877666.429960:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877666.429969:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877666.429975:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877666/real 1788877666] req@ffffa061888af480 x1875771511402752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877666.429987:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511402752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877666.429992:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877666.429994:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877666.429996:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877666.429999:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877666.430008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877666.430013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877666/real 1788877666] req@ffffa061888ae300 x1875771511402880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877666.430020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511402880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877666.430025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877666.430027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877666.430029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511402880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877667.069453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877667.069661:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877668.093489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788877668.093567:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877669.117483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877670.141537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877670.141737:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1788877671.165490:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877671.165626:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:3.0:1788877671.549579:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877671.549593:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877671.549599:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877671.549611:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2658 00000100:00080000:3.0:1788877671.549623:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877671.549631:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877671.549674:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511403008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877671.549710:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877671.549714:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877671.549716:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877671.549721:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2658 00000100:00080000:3.0:1788877671.549725:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877671.549730:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877671.549741:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877671.549756:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511403136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877671.549769:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511403008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877671.549774:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877671.549783:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877671.549787:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877671.549795:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2658 00000100:00080000:3.0:1788877671.549801:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877671.549808:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877671.549818:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877671.549839:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511403264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877671.549855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877671.549866:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877671.549874:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877671.549877:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877671.549885:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2658 00000100:00080000:3.0:1788877671.549891:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877671.549897:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877671.549905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877671/real 1788877671] req@ffffa06188871500 x1875771511403008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788877671.549906:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877671.549931:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511403392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877671.549939:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511403008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877671.549954:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877671.549964:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877671.549967:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877671.549968:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877671.549974:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2658 00000100:00080000:2.0:1788877671.549982:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788877671.549984:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877671.549995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511403008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877671.549995:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877671.550001:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877671.550016:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511403520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877671.550032:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877671.550063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877671.550077:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877671/real 1788877671] req@ffffa06188873100 x1875771511403136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877671.550091:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511403136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877671.550108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877671.550111:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877671.550115:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877671.550121:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877671.550142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877671.550160:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877671/real 1788877671] req@ffffa06188872a00 x1875771511403264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877671.550180:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511403264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877671.550195:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877671.550200:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877671.550206:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877671.550218:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877671.550251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877671.550265:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877671/real 1788877671] req@ffffa06188873800 x1875771511403392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877671.550278:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511403392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877671.550287:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877671.550290:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877671.550294:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877671.550313:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877671.550331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877671.550341:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877671/real 1788877671] req@ffffa06188873480 x1875771511403520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877671.550354:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511403520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877671.550362:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877671.550365:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877671.550368:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877672.189477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877673.213551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877673.213656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877674.237513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788877674.237569:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877675.261538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877676.285539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877676.285603:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788877676.669622:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511403648:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788877676.669694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877676.669740:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877676/real 1788877676] req@ffffa06198afe300 x1875771511403648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877676.669761:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511403648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877676.669774:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877676.669779:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877676.669784:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511403648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877676.669808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877676.669828:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877676.669840:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877676/real 1788877676] req@ffffa06198afca80 x1875771511403776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877676.669852:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511403776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877676.669860:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877676.669864:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877676.669867:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877676.669873:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877676.669889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877676.669898:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877676/real 1788877676] req@ffffa06198afd500 x1875771511403904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877676.669908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511403904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877676.669915:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877676.669918:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877676.669922:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511403904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877676.669928:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877676.669943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877676.669952:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877676/real 1788877676] req@ffffa06198afd180 x1875771511404032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877676.669969:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511404032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877676.669976:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877676.669979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877676.669982:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877676.669988:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877676.670004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877676.670015:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877676/real 1788877676] req@ffffa06198afc380 x1875771511404160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877676.670025:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511404160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877676.670033:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877676.670036:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877676.670038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877677.309497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788877677.309528:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877678.333499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877679.357511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877679.357573:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877680.381533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788877680.381542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877681.405509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:2.0:1788877681.789486:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877681.789494:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877681.789498:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877681.789507:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2668 00000100:00080000:2.0:1788877681.789517:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877681.789523:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877681.789555:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511404288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877681.789579:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877681.789582:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877681.789583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877681.789586:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2668 00000100:00080000:2.0:1788877681.789588:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877681.789590:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877681.789597:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877681.789606:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511404416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877681.789613:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877681.789616:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877681.789617:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877681.789620:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2668 00000100:00080000:2.0:1788877681.789622:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877681.789624:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877681.789627:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877681.789633:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511404544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877681.789638:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511404288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877681.789643:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877681.789647:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877681.789648:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877681.789651:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2668 00000100:00080000:2.0:1788877681.789653:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877681.789655:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877681.789659:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877681.789667:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511404672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877681.789673:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877681.789675:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877681.789676:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877681.789679:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2668 00000100:00080000:2.0:1788877681.789690:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877681.789692:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877681.789696:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877681.789700:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877681.789706:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511404800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877681.789734:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877681/real 1788877681] req@ffffa061888ae680 x1875771511404288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877681.789748:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511404288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877681.789756:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877681.789759:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877681.789762:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511404288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877681.789767:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877681.789792:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877681.789800:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877681/real 1788877681] req@ffffa061888afb80 x1875771511404416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877681.789808:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511404416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877681.789813:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877681.789815:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877681.789817:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877681.789839:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877681.789858:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877681.789868:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877681/real 1788877681] req@ffffa061888af100 x1875771511404544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877681.789880:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511404544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877681.789887:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877681.789890:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877681.789904:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877681.789911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877681.789930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877681.789941:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877681/real 1788877681] req@ffffa061888adc00 x1875771511404672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877681.789953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511404672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877681.789972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877681.789975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877681.789978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877681.789984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877681.790001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877681.790011:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877681/real 1788877681] req@ffffa061888aca80 x1875771511404800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877681.790023:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511404800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877681.790042:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877681.790045:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877681.790048:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511404800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877682.429477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877682.429533:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877683.453591:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877683.453667:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877684.477562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877685.501492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877685.501555:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877686.525498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877686.525559:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877686.909489:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877686.909499:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877686.909507:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877686.909518:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2673 00000100:00080000:2.0:1788877686.909529:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877686.909537:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877686.909574:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511404928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877686.909595:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877686.909598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877686.909599:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877686.909603:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2673 00000100:00080000:2.0:1788877686.909606:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877686.909610:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877686.909618:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877686.909628:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511404928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877686.909630:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511405056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877686.909643:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877686.909645:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877686.909647:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877686.909649:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2673 00000100:00080000:2.0:1788877686.909651:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877686.909654:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877686.909657:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877686.909664:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511405184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877686.909670:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877686.909673:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877686.909674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877686.909677:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2673 00000100:00080000:2.0:1788877686.909681:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877686.909684:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877686.909687:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877686.909695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877686.909696:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511405312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877686.909705:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877686.909707:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877686.909709:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877686.909711:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2673 00000100:00080000:2.0:1788877686.909714:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877686.909716:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877686.909719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877686.909727:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511405440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877686.909731:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877686/real 1788877686] req@ffffa061888ad880 x1875771511404928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877686.909749:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511404928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877686.909757:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877686.909760:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877686.909762:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511404928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877686.909779:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877686.909794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877686.909803:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877686/real 1788877686] req@ffffa061888ac380 x1875771511405056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877686.909811:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511405056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877686.909817:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877686.909820:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877686.909822:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877686.909825:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877686.909836:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877686.909842:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877686/real 1788877686] req@ffffa061888aea00 x1875771511405184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877686.909850:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511405184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877686.909860:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877686.909862:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877686.909864:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877686.909867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877686.909877:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877686.909883:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877686/real 1788877686] req@ffffa061888ac700 x1875771511405312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877686.909890:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511405312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877686.909896:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877686.909897:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877686.909899:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877686.909902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877686.909912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877686.909918:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877686/real 1788877686] req@ffffa061888aed80 x1875771511405440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877686.909925:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511405440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877686.909930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877686.909932:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877686.909935:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877687.549478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877688.573479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877688.573511:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877689.597551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877689.597643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877690.621548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877691.645498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877691.645616:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877692.029555:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877692.029568:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877692.029575:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877692.029608:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2678 00000100:00080000:3.0:1788877692.029623:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877692.029633:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877692.029681:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511405568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877692.029719:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877692.029724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877692.029726:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877692.029732:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2678 00000100:00080000:3.0:1788877692.029737:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877692.029742:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877692.029763:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877692.029785:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511405696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877692.029789:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511405568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877692.029810:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877692.029816:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877692.029818:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877692.029822:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2678 00000100:00080000:3.0:1788877692.029827:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877692.029836:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877692.029847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877692.029872:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511405824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877692.029892:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1788877692.029900:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877692.029905:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877692.029913:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877692.029924:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2678 00000100:00080000:3.0:1788877692.029933:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877692.029941:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877692.029950:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877692.029956:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877692/real 1788877692] req@ffffa06188872300 x1875771511405568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877692.029975:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511405952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877692.029992:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511405568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877692.029995:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877692.030012:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877692.030015:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877692.030021:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2678 00000100:00080000:3.0:1788877692.030026:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877692.030036:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877692.030047:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1788877692.030049:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877692.030059:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877692.030065:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511406080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877692.030074:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511405568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877692.030124:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877692.030158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877692.030174:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877692/real 1788877692] req@ffffa06188873b80 x1875771511405696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877692.030190:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511405696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877692.030201:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877692.030206:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877692.030210:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877692.030217:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877692.030236:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877692.030248:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877692/real 1788877692] req@ffffa06188870700 x1875771511405824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877692.030264:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511405824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877692.030280:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877692.030286:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877692.030293:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877692.030305:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877692.030346:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877692.030368:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877692/real 1788877692] req@ffffa06188872680 x1875771511405952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877692.030449:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511405952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877692.030462:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877692.030467:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877692.030472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511405952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877692.030480:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877692.030510:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877692.030525:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877692/real 1788877692] req@ffffa06188871c00 x1875771511406080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877692.030540:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511406080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877692.030549:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877692.030553:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877692.030556:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877692.669464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877692.669542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877693.693502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877694.717469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877694.717528:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877695.741535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877695.741703:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877696.765484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:2.0:1788877697.149521:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877697.149532:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877697.149536:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877697.149549:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2683 00000100:00080000:2.0:1788877697.149562:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877697.149570:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877697.149621:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511406208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877697.149645:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877697.149648:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877697.149650:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877697.149653:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2683 00000100:00080000:2.0:1788877697.149656:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877697.149659:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877697.149668:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877697.149678:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511406336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877697.149687:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877697.149690:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877697.149691:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877697.149696:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2683 00000100:00080000:2.0:1788877697.149699:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788877697.149700:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511406208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877697.149703:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877697.149710:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877697.149719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511406464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877697.149726:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877697.149728:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877697.149730:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877697.149732:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2683 00000100:00080000:2.0:1788877697.149734:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877697.149736:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877697.149739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877697.149747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511406592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877697.149753:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877697.149755:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877697.149756:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877697.149758:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2683 00000100:00080000:2.0:1788877697.149768:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877697.149772:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877697.149776:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877697.149785:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511406720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788877697.149790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877697.149836:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877697/real 1788877697] req@ffffa061888ace00 x1875771511406208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877697.149857:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511406208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877697.149870:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877697.149874:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877697.149879:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511406208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877697.149906:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877697.149930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877697.149942:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877697/real 1788877697] req@ffffa061888ad180 x1875771511406336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877697.149954:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511406336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877697.149964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877697.149967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877697.149970:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877697.149977:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877697.149996:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877697.150006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877697/real 1788877697] req@ffffa061888aed80 x1875771511406464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877697.150024:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511406464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877697.150032:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877697.150036:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877697.150039:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877697.150045:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877697.150064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877697.150073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877697/real 1788877697] req@ffffa061888ac700 x1875771511406592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877697.150085:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511406592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877697.150093:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877697.150097:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877697.150100:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877697.150106:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877697.150124:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877697.150133:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877697/real 1788877697] req@ffffa061888aea00 x1875771511406720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877697.150144:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511406720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877697.150159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877697.150163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877697.150166:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877697.789460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877697.789565:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877698.813515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877698.813571:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877699.837479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877700.861548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877700.861659:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877701.885497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877701.885611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877702.269589:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877702.269603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877702.269610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877702.269626:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2688 00000100:00080000:2.0:1788877702.269642:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877702.269654:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877702.269707:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511406848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877702.269747:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877702.269752:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877702.269754:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877702.269760:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2688 00000100:00080000:2.0:1788877702.269764:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877702.269769:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877702.269781:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877702.269798:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511406976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877702.269811:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877702.269816:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877702.269819:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877702.269824:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2688 00000100:00080000:2.0:1788877702.269829:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877702.269838:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877702.269838:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511406848:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788877702.269854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877702.269873:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511407104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877702.269889:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877702.269894:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877702.269897:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877702.269903:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2688 00000100:00080000:2.0:1788877702.269919:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877702.269924:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877702.269934:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788877702.269940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877702.269969:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511407232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877702.269992:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788877702.269995:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877702/real 1788877702] req@ffffa061888ac380 x1875771511406848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877702.270004:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877702.270014:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877702.270030:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2688 00000100:00100000:3.0:1788877702.270037:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511406848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877702.270043:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877702.270062:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788877702.270064:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877702.270078:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:2.0:1788877702.270080:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877702.270085:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511406848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877702.270101:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877702.270115:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511407360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788877702.270135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877702.270157:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877702/real 1788877702] req@ffffa061888ad880 x1875771511406976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877702.270174:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511406976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877702.270185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877702.270189:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877702.270193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511406976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877702.270232:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877702.270273:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877702.270295:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877702/real 1788877702] req@ffffa061888aca80 x1875771511407104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877702.270321:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511407104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877702.270339:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877702.270345:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877702.270351:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877702.270364:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877702.270496:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877702.270517:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877702/real 1788877702] req@ffffa061888adc00 x1875771511407232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877702.270532:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511407232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877702.270544:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877702.270548:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877702.270552:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877702.270565:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877702.270592:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877702.270605:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877702/real 1788877702] req@ffffa061888af100 x1875771511407360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877702.270619:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511407360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877702.270629:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877702.270632:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877702.270636:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877702.909496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877703.933523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877703.933628:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877704.957560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877704.957669:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877705.981458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877707.005452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877707.005501:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877707.389486:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877707.389494:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877707.389498:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877707.389506:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2694 00000100:00080000:3.0:1788877707.389514:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877707.389520:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877707.389552:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511407488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877707.389575:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877707.389577:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877707.389579:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877707.389581:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2694 00000100:00080000:3.0:1788877707.389583:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877707.389586:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877707.389593:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877707.389602:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511407616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877707.389615:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877707.389618:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511407488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877707.389619:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877707.389622:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877707.389630:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2694 00000100:00080000:3.0:1788877707.389635:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877707.389637:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877707.389641:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877707.389649:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511407744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877707.389655:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877707.389658:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877707.389660:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877707.389662:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2694 00000100:00080000:3.0:1788877707.389664:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877707.389666:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877707.389669:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877707.389676:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511407872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877707.389681:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877707.389684:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877707.389685:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877707.389688:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2694 00000100:00080000:3.0:1788877707.389691:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877707.389694:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877707.389697:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877707.389705:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511408000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877707.389736:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877707.389771:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877707.389799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877707.389826:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877708.029466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877708.029565:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877709.053522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877710.077540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877710.077646:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788877710.461825:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877710.461899:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877710.461926:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877710.461990:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877710.462009:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877710.462025:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877710.462042:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788877710.462132:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877707/real 1788877710] req@ffffa06188870380 x1875771511407616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877710.462187:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511407616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877710.462224:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877710.462232:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877710.462239:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877710.462253:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877707/real 1788877710] req@ffffa06188870a80 x1875771511407744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877710.462276:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511407744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877710.462290:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877710.462295:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877710.462301:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877710.462319:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877707/real 1788877710] req@ffffa06188871880 x1875771511407872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877710.462433:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511407872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877710.462450:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877710.462457:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877710.462463:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511407872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877710.462476:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877707/real 1788877710] req@ffffa06188871c00 x1875771511408000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877710.462497:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511408000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877710.462514:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877710.462522:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877710.462529:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877710.462545:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877707/real 1788877710] req@ffffa06188872d80 x1875771511407488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877710.462582:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511407488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877710.462596:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877710.462602:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877710.462607:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511407488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877711.101517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877711.101687:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877712.125534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:3.0:1788877712.509518:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877712.509530:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877712.509536:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877712.509559:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2699 00000100:00080000:3.0:1788877712.509572:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877712.509581:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877712.509624:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511408128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877712.509654:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877712.509658:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877712.509665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877712.509673:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2699 00000100:00080000:3.0:1788877712.509678:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877712.509682:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877712.509694:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877712.509717:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511408128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877712.509720:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511408256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877712.509743:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877712.509752:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877712.509755:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877712.509762:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2699 00000100:00080000:3.0:1788877712.509782:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877712.509790:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788877712.509797:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788877712.509803:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877712.509828:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511408384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877712.509840:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877712/real 1788877712] req@ffffa06188872680 x1875771511408128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877712.509849:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877712.509865:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877712.509871:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788877712.509879:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511408128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877712.509883:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2699 00000100:00080000:3.0:1788877712.509895:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877712.509907:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877712.509916:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:2.0:1788877712.509922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788877712.509931:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877712.509934:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511408128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877712.509960:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511408512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877712.509978:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408256:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788877712.509980:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877712.509992:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877712.509995:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877712.510002:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2699 00000100:00080000:3.0:1788877712.510009:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877712.510017:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877712.510027:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877712.510034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877712.510055:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511408640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877712.510066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877712/real 1788877712] req@ffffa06188870700 x1875771511408256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877712.510098:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511408256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877712.510113:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877712.510120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877712.510126:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877712.510136:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877712.510181:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877712.510203:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877712/real 1788877712] req@ffffa06188873b80 x1875771511408384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877712.510225:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511408384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877712.510238:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877712.510244:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877712.510250:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877712.510276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877712.510312:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877712.510354:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877712/real 1788877712] req@ffffa06188872300 x1875771511408512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877712.510376:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511408512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877712.510444:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877712.510451:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877712.510458:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877712.510468:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877712.510506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877712.510526:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877712/real 1788877712] req@ffffa06188873480 x1875771511408640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877712.510547:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511408640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877712.510563:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877712.510568:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877712.510574:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877713.149507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877713.149597:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877714.173592:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877714.173664:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877715.197550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877716.221501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877716.221627:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877717.245461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877717.245517:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877717.629455:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877717.629465:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877717.629469:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877717.629479:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2704 00000100:00080000:3.0:1788877717.629489:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877717.629497:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877717.629532:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511408768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877717.629554:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877717.629557:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877717.629559:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877717.629562:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2704 00000100:00080000:3.0:1788877717.629565:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877717.629567:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877717.629575:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877717.629584:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511408896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877717.629591:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877717.629593:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877717.629594:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877717.629596:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2704 00000100:00080000:3.0:1788877717.629598:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877717.629601:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877717.629605:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877717.629617:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511409024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877717.629628:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511408768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877717.629637:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877717.629643:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877717.629645:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877717.629650:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2704 00000100:00080000:3.0:1788877717.629655:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877717.629660:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877717.629666:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877717.629677:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511409152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877717.629684:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1788877717.629687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877717.629689:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877717.629692:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877717.629696:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2704 00000100:00080000:3.0:1788877717.629700:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877717.629703:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877717.629707:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877717.629717:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511409280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877717.629723:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877717/real 1788877717] req@ffffa06188873800 x1875771511408768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877717.629762:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511408768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877717.629771:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877717.629774:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877717.629777:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511408768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877717.629783:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877717.629798:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877717.629806:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877717/real 1788877717] req@ffffa06188872a00 x1875771511408896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877717.629826:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511408896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877717.629837:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877717.629840:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877717.629842:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511408896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877717.629859:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877717.629869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877717.629875:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877717/real 1788877717] req@ffffa06188873100 x1875771511409024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877717.629882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511409024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877717.629887:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877717.629889:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877717.629891:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877717.629895:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877717.629905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877717.629911:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877717/real 1788877717] req@ffffa06188871500 x1875771511409152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877717.629918:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511409152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877717.629922:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877717.629924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877717.629927:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877717.629930:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877717.629940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877717.629946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877717/real 1788877717] req@ffffa06188873480 x1875771511409280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877717.629953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511409280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877717.629959:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877717.629961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877717.629965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877718.269515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877719.293471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877719.293516:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877720.317513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877720.317579:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877721.341467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877722.365508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877722.365628:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877722.749557:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877722.749571:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877722.749576:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877722.749591:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2709 00000100:00080000:2.0:1788877722.749606:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877722.749616:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877722.749663:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511409408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877722.749703:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877722.749708:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877722.749712:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877722.749718:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2709 00000100:00080000:2.0:1788877722.749723:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877722.749729:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877722.749742:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877722.749759:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511409536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877722.749776:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511409408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877722.749778:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877722.749794:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877722.749797:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877722.749803:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2709 00000100:00080000:2.0:1788877722.749806:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877722.749810:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877722.749816:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877722.749832:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511409664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877722.749845:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877722.749851:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877722.749855:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877722.749864:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2709 00000100:00080000:2.0:1788877722.749872:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788877722.749881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877722.749885:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877722.749900:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877722.749928:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511409792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877722.749938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877722/real 1788877722] req@ffffa061888afb80 x1875771511409408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877722.749974:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877722.749981:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877722.749985:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788877722.749993:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511409408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877722.749997:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2709 00000100:00080000:2.0:1788877722.750010:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877722.750024:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788877722.750025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788877722.750037:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788877722.750039:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877722.750052:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511409408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877722.750066:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511409920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788877722.750092:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877722.750125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877722.750141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877722/real 1788877722] req@ffffa061888ae680 x1875771511409536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877722.750156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511409536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877722.750166:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877722.750170:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877722.750174:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877722.750182:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788877722.750202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877722.750214:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877722/real 1788877722] req@ffffa061888ae300 x1875771511409664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877722.750228:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511409664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877722.750238:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877722.750242:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877722.750245:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877722.750252:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877722.750272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877722.750319:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877722/real 1788877722] req@ffffa061888af480 x1875771511409792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877722.750347:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511409792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877722.750364:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877722.750371:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877722.750378:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788877722.750427:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788877722.750459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788877722.750474:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877722/real 1788877722] req@ffffa061888ac000 x1875771511409920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877722.750489:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511409920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877722.750501:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877722.750504:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877722.750508:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511409920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877723.389473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877723.389568:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877724.413518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877725.437480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877725.437544:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877726.461450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877726.461504:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877727.485488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:3.0:1788877727.869588:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877727.869601:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877727.869608:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877727.869621:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2714 00000100:00080000:3.0:1788877727.869634:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877727.869643:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877727.869686:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511410048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877727.869721:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877727.869725:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877727.869728:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877727.869732:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2714 00000100:00080000:3.0:1788877727.869736:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877727.869740:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877727.869751:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877727.869765:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511410176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877727.869780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788877727.869782:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511410048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877727.869786:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877727.869815:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877727.869824:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2714 00000100:00080000:3.0:1788877727.869828:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877727.869834:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877727.869841:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877727.869864:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511410304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877727.869885:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877727.869891:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877727.869896:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788877727.869902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877727.869912:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2714 00000100:00080000:3.0:1788877727.869924:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877727.869934:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877727.869947:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877727.869957:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877727/real 1788877727] req@ffffa06188872300 x1875771511410048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877727.869975:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511410432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877727.869991:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511410048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877727.869994:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877727.870012:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877727.870018:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877727.870046:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877727.870047:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2714 00000100:00080000:3.0:1788877727.870060:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877727.870061:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788877727.870073:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877727.870075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511410048:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788877727.870085:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877727.870104:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511410560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877727.870116:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877727.870150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877727.870185:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877727/real 1788877727] req@ffffa06188873b80 x1875771511410176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877727.870201:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511410176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877727.870212:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877727.870216:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877727.870220:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877727.870227:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877727.870251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877727.870309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877727/real 1788877727] req@ffffa06188870700 x1875771511410304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877727.870335:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511410304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877727.870344:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877727.870348:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877727.870351:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877727.870377:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877727.870405:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877727.870454:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877727/real 1788877727] req@ffffa06188872680 x1875771511410432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877727.870471:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511410432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877727.870481:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877727.870485:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877727.870489:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877727.870510:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877727.870551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877727.870563:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877727/real 1788877727] req@ffffa06188872d80 x1875771511410560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877727.870575:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511410560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877727.870608:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877727.870615:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877727.870621:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511410560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788877728.509473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877728.509532:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877729.533496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877729.533592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877730.557483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877731.581501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877731.581587:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877732.605516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877732.605715:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877732.989500:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877732.989510:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877732.989514:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877732.989525:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2719 00000100:00080000:3.0:1788877732.989534:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877732.989542:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877732.989578:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511410688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877732.989623:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877732.989627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877732.989629:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877732.989633:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2719 00000100:00080000:3.0:1788877732.989638:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877732.989642:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877732.989652:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877732.989686:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511410816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877732.989696:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877732.989699:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877732.989700:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877732.989703:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2719 00000100:00080000:3.0:1788877732.989706:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877732.989711:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877732.989716:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877732.989726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511410944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877732.989734:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877732.989737:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877732.989739:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877732.989749:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2719 00000100:00080000:3.0:1788877732.989753:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877732.989756:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877732.989772:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877732.989782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511411072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877732.989790:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877732.989795:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877732.989799:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877732.989812:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2719 00000100:00080000:3.0:1788877732.989819:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877732.989830:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877732.989842:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877732.989864:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511411200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877733.629466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877734.653469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877734.653530:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877735.677471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877735.677529:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877736.701466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877737.725519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877737.725613:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877738.109492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877738.109500:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877738.109504:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877738.109523:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2724 00000100:00080000:1.0:1788877738.109532:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877738.109538:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877738.109571:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511411328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877738.109591:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877738.109594:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877738.109596:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877738.109598:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2724 00000100:00080000:1.0:1788877738.109601:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877738.109603:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877738.109613:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877738.109628:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511411456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877738.109643:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877738.109647:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877738.109649:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877738.109654:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2724 00000100:00080000:1.0:1788877738.109658:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877738.109662:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877738.109667:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877738.109676:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511411584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877738.109683:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877738.109689:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877738.109691:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877738.109697:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2724 00000100:00080000:1.0:1788877738.109700:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877738.109702:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877738.109706:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877738.109714:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511411712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877738.109721:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877738.109723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877738.109725:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877738.109728:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2724 00000100:00080000:1.0:1788877738.109731:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877738.109736:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877738.109742:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877738.109753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511411840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788877738.749478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877738.749511:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877739.773594:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788877740.797541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877740.797628:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877741.821513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877741.821705:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788877742.845467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00080000:1.0:1788877743.229481:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877743.229490:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877743.229495:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877743.229505:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2729 00000100:00080000:1.0:1788877743.229517:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877743.229529:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877743.229569:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511411968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877743.229593:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877743.229596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877743.229597:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877743.229601:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2729 00000100:00080000:1.0:1788877743.229604:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877743.229607:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877743.229616:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877743.229626:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511412096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877743.229637:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877743.229645:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877743.229648:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877743.229655:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2729 00000100:00080000:1.0:1788877743.229663:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877743.229665:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877743.229669:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877743.229680:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511412224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877743.229688:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877743.229691:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877743.229692:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877743.229694:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2729 00000100:00080000:1.0:1788877743.229697:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877743.229701:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877743.229704:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877743.229713:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511412352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877743.229719:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877743.229721:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877743.229723:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877743.229725:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2729 00000100:00080000:1.0:1788877743.229728:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877743.229730:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877743.229739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877743.229763:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511412480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877743.869532:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877744.893591:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877745.917640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877746.941588:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877747.965539:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877748.349510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877748.349537:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877748.349542:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877748.349565:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2735 00000100:00080000:2.0:1788877748.349577:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877748.349586:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877748.349644:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511412608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877748.349689:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877748.349695:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877748.349698:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877748.349704:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2735 00000100:00080000:2.0:1788877748.349718:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877748.349727:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877748.349740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877748.349761:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511412736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877748.349777:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877748.349782:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877748.349786:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877748.349790:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2735 00000100:00080000:2.0:1788877748.349796:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877748.349806:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877748.349813:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877748.349844:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511412864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877748.349857:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877748.349865:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877748.349870:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877748.349876:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2735 00000100:00080000:2.0:1788877748.349882:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877748.349890:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877748.349914:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877748.349936:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511412992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877748.349953:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877748.349976:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877748.349979:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877748.349985:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2735 00000100:00080000:2.0:1788877748.349990:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877748.350000:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877748.350006:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877748.350037:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511413120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877748.989583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877750.013617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877751.037596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877752.061642:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877753.085541:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877753.469547:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877753.469561:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877753.469568:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877753.469583:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2740 00000100:00080000:2.0:1788877753.469597:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877753.469609:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877753.469656:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511413248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877753.469697:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877753.469703:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877753.469706:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877753.469714:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2740 00000100:00080000:2.0:1788877753.469722:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877753.469730:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877753.469743:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877753.469774:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511413376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877753.469828:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877753.469835:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877753.469838:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877753.469845:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2740 00000100:00080000:2.0:1788877753.469856:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877753.469867:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877753.469876:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877753.469918:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511413504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877753.469957:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877753.469965:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877753.469972:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877753.469986:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2740 00000100:00080000:2.0:1788877753.469996:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877753.470013:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877753.470026:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877753.470051:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511413632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877753.470073:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877753.470083:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877753.470089:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877753.470096:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2740 00000100:00080000:2.0:1788877753.470106:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877753.470121:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877753.470134:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877753.470199:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511413760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877754.109575:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877755.133708:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877756.157606:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877757.181576:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877758.205572:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877758.589538:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877758.589547:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877758.589551:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877758.589560:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2745 00000100:00080000:1.0:1788877758.589569:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877758.589576:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877758.589610:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511413888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877758.589632:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877758.589635:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877758.589636:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877758.589640:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2745 00000100:00080000:1.0:1788877758.589642:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877758.589646:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877758.589654:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877758.589665:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511414016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877758.589681:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877758.589688:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877758.589690:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877758.589692:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2745 00000100:00080000:1.0:1788877758.589695:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877758.589697:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877758.589701:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877758.589712:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511414144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877758.589738:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877758.589742:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877758.589743:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877758.589748:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2745 00000100:00080000:1.0:1788877758.589752:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877758.589756:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877758.589762:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877758.589779:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511414272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877758.589787:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877758.589790:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877758.589802:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877758.589806:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2745 00000100:00080000:1.0:1788877758.589810:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877758.589817:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877758.589825:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877758.589836:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511414400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877759.229639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877760.253614:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877761.277516:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877762.301600:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877763.325550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877763.709558:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877763.709575:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877763.709583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877763.709600:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2750 00000100:00080000:1.0:1788877763.709618:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877763.709630:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877763.709696:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511414528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877763.709745:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877763.709751:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877763.709754:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877763.709760:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2750 00000100:00080000:1.0:1788877763.709765:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877763.709769:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877763.709784:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877763.709800:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511414656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877763.709816:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877763.709821:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877763.709823:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877763.709829:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2750 00000100:00080000:1.0:1788877763.709833:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877763.709838:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877763.709845:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877763.709862:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511414784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877763.709886:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877763.709890:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877763.709892:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877763.709897:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2750 00000100:00080000:1.0:1788877763.709901:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877763.709905:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877763.709911:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877763.709928:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511414912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877763.709940:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877763.709944:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877763.709946:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877763.709950:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2750 00000100:00080000:1.0:1788877763.709968:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877763.709973:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877763.709980:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877763.710006:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511415040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877764.349629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877765.373620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877766.397600:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877767.421599:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877768.445545:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877768.829556:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877768.829572:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877768.829579:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877768.829597:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2755 00000100:00080000:1.0:1788877768.829613:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877768.829624:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877768.829687:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511415168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877768.829734:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877768.829754:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877768.829756:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877768.829761:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2755 00000100:00080000:1.0:1788877768.829764:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877768.829768:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877768.829779:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877768.829794:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511415296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877768.829840:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877768.829846:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877768.829849:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877768.829856:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2755 00000100:00080000:1.0:1788877768.829863:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877768.829888:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877768.829899:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877768.829927:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511415424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877768.829985:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877768.829989:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877768.829992:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877768.829999:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2755 00000100:00080000:1.0:1788877768.830005:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877768.830012:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877768.830019:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877768.830040:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511415552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877768.830066:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877768.830069:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877768.830072:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877768.830098:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2755 00000100:00080000:1.0:1788877768.830107:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877768.830113:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877768.830118:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877768.830138:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511415680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877769.469592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877770.493685:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877771.517667:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877772.541672:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877773.565586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877773.949525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877773.949539:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877773.949545:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877773.949559:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2760 00000100:00080000:1.0:1788877773.949577:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877773.949588:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877773.949648:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511415808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877773.949712:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877773.949719:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877773.949724:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877773.949734:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2760 00000100:00080000:1.0:1788877773.949744:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877773.949755:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877773.949772:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877773.949802:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511415936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877773.949820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877773.949823:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877773.949827:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877773.949833:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2760 00000100:00080000:1.0:1788877773.949858:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877773.949865:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877773.949873:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877773.949904:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511416064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877773.949925:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877773.949932:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877773.949935:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877773.949942:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2760 00000100:00080000:1.0:1788877773.949948:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877773.949955:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877773.949965:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877773.949987:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511416192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877773.950007:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877773.950012:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877773.950015:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877773.950021:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2760 00000100:00080000:1.0:1788877773.950029:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877773.950043:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877773.950074:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877773.950099:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511416320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877774.589665:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877775.613557:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877776.637507:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788877777.021696:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788877777.021747:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788877777.021765:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877777.021804:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877777.021813:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877777.021820:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788877777.021826:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00010000:00010000:2.0:1788877777.661702:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877778.685638:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877779.069607:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877779.069621:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877779.069629:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877779.069659:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2765 00000100:00080000:2.0:1788877779.069671:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877779.069680:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877779.069715:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511416448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877779.069767:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877779.069772:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877779.069775:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877779.069781:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2765 00000100:00080000:2.0:1788877779.069785:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877779.069790:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877779.069818:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877779.069831:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511416576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877779.069841:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877779.069845:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877779.069847:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877779.069850:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2765 00000100:00080000:2.0:1788877779.069853:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877779.069857:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877779.069864:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877779.069876:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511416704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877779.069892:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877779.069895:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877779.069897:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877779.069899:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2765 00000100:00080000:2.0:1788877779.069902:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877779.069905:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877779.069909:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877779.069919:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511416832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877779.069927:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877779.069930:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877779.069932:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877779.069935:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2765 00000100:00080000:2.0:1788877779.069938:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877779.069941:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877779.069944:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877779.069957:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511416960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877779.709618:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877780.733546:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877781.757603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877782.781570:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877783.805622:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877784.189573:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877784.189589:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877784.189598:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877784.189616:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2770 00000100:00080000:2.0:1788877784.189633:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877784.189644:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877784.189692:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511417088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877784.189730:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877784.189735:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877784.189738:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877784.189744:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2770 00000100:00080000:2.0:1788877784.189748:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877784.189754:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877784.189767:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877784.189794:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511417216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877784.189824:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877784.189829:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877784.189832:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877784.189838:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2770 00000100:00080000:2.0:1788877784.189842:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877784.189847:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877784.189854:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877784.189873:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511417344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877784.189887:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877784.189891:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877784.189894:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877784.189898:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2770 00000100:00080000:2.0:1788877784.189902:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877784.189915:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877784.189928:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877784.189944:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511417472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877784.189956:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877784.189960:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877784.189963:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877784.189970:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2770 00000100:00080000:2.0:1788877784.189977:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877784.190027:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877784.190038:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877784.190074:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511417600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877784.829676:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877785.853610:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877786.877550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877787.901495:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877788.925696:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877789.309544:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877789.309561:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877789.309568:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877789.309583:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2776 00000100:00080000:3.0:1788877789.309601:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877789.309611:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877789.309683:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511417728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877789.309711:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877789.309715:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877789.309717:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877789.309722:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2776 00000100:00080000:3.0:1788877789.309725:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877789.309730:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877789.309760:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877789.309779:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511417856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877789.309793:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877789.309798:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877789.309800:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877789.309805:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2776 00000100:00080000:3.0:1788877789.309809:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877789.309815:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877789.309821:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877789.309848:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511417984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877789.309860:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877789.309865:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877789.309868:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877789.309873:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2776 00000100:00080000:3.0:1788877789.309877:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877789.309883:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877789.309889:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877789.309902:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511418112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877789.309914:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877789.309918:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877789.309921:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877789.309925:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2776 00000100:00080000:3.0:1788877789.309930:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877789.309935:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877789.309943:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877789.309956:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511418240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877789.949611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877790.973542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877791.997534:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877793.021653:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877794.045671:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877794.429576:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877794.429593:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877794.429601:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877794.429618:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2781 00000100:00080000:2.0:1788877794.429638:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877794.429650:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877794.429717:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511418368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877794.429759:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877794.429765:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877794.429768:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877794.429774:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2781 00000100:00080000:2.0:1788877794.429779:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877794.429785:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877794.429802:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877794.429837:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511418496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877794.429866:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877794.429873:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877794.429879:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877794.429888:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2781 00000100:00080000:2.0:1788877794.429897:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877794.429906:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877794.429918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877794.429948:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511418624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877794.429998:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877794.430003:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877794.430007:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877794.430014:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2781 00000100:00080000:2.0:1788877794.430020:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877794.430028:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877794.430038:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877794.430060:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511418752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877794.430102:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877794.430107:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877794.430110:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877794.430121:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2781 00000100:00080000:2.0:1788877794.430132:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877794.430144:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877794.430158:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877794.430179:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511418880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877795.069666:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877796.093572:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877797.117518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877798.141517:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877799.165506:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877799.549555:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877799.549563:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877799.549567:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877799.549579:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2786 00000100:00080000:3.0:1788877799.549591:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877799.549599:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877799.549642:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511419008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877799.549669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877799.549672:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877799.549674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877799.549677:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2786 00000100:00080000:3.0:1788877799.549681:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877799.549685:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877799.549694:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877799.549708:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511419136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877799.549720:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877799.549724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877799.549726:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877799.549729:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2786 00000100:00080000:3.0:1788877799.549732:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877799.549735:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877799.549739:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877799.549750:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511419264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877799.549757:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877799.549760:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877799.549761:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877799.549764:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2786 00000100:00080000:3.0:1788877799.549767:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877799.549769:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877799.549773:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877799.549783:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511419392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877799.549792:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877799.549795:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877799.549797:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877799.549800:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2786 00000100:00080000:3.0:1788877799.549802:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877799.549805:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877799.549829:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877799.549849:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511419520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877800.189621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877801.213653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877802.237714:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877803.261706:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877804.285618:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877804.669563:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877804.669579:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877804.669587:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877804.669604:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2791 00000100:00080000:1.0:1788877804.669624:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877804.669636:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877804.669705:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511419648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877804.669748:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877804.669753:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877804.669756:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877804.669763:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2791 00000100:00080000:1.0:1788877804.669767:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877804.669773:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877804.669789:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877804.669808:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511419776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877804.669828:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877804.669840:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877804.669842:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877804.669848:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2791 00000100:00080000:1.0:1788877804.669851:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877804.669856:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877804.669861:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877804.669880:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511419904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877804.669892:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877804.669897:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877804.669899:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877804.669904:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2791 00000100:00080000:1.0:1788877804.669956:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877804.669968:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877804.669986:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877804.670023:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511420032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877804.670045:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877804.670055:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877804.670063:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877804.670080:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2791 00000100:00080000:1.0:1788877804.670090:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877804.670097:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877804.670108:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877804.670137:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511420160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877805.309583:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877806.333546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877807.357542:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877808.381553:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877809.405574:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877809.789545:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877809.789578:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877809.789585:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877809.789606:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2796 00000100:00080000:1.0:1788877809.789626:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877809.789639:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877809.789714:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511420288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877809.789747:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877809.789755:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877809.789758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877809.789763:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2796 00000100:00080000:1.0:1788877809.789770:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877809.789775:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877809.789810:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877809.789842:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511420416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877809.789877:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877809.789881:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877809.789884:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877809.789909:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2796 00000100:00080000:1.0:1788877809.789920:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877809.789945:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877809.789954:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877809.789965:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511420544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877809.789974:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877809.789977:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877809.789980:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877809.789984:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2796 00000100:00080000:1.0:1788877809.789989:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877809.789992:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877809.790002:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877809.790020:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511420672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877809.790070:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877809.790086:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877809.790088:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877809.790093:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2796 00000100:00080000:1.0:1788877809.790095:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877809.790099:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877809.790105:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877809.790120:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511420800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877810.429575:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877811.453713:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877812.477627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877813.501699:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877814.525573:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877814.909519:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877814.909535:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877814.909541:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877814.909557:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2801 00000100:00080000:3.0:1788877814.909574:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877814.909588:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877814.909647:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511420928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877814.909684:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877814.909689:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877814.909692:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877814.909700:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2801 00000100:00080000:3.0:1788877814.909706:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877814.909713:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877814.909738:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877814.909761:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511421056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877814.909773:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877814.909777:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877814.909779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877814.909783:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2801 00000100:00080000:3.0:1788877814.909787:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877814.909791:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877814.909797:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877814.909808:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511421184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877814.909825:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877814.909829:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877814.909832:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877814.909836:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2801 00000100:00080000:3.0:1788877814.909840:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877814.909844:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877814.909848:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877814.909857:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511421312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877814.909883:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877814.909888:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877814.909893:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877814.909921:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2801 00000100:00080000:3.0:1788877814.909928:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877814.909935:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877814.909942:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877814.909967:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511421440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877815.549660:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877816.573596:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877817.597590:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877818.621519:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877819.645592:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877820.029585:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877820.029616:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877820.029622:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877820.029636:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2806 00000100:00080000:2.0:1788877820.029651:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877820.029661:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877820.029710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511421568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877820.029750:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877820.029755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877820.029758:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877820.029764:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2806 00000100:00080000:2.0:1788877820.029768:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877820.029773:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877820.029798:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877820.029823:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511421696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877820.029905:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877820.029910:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877820.029912:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877820.029917:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2806 00000100:00080000:2.0:1788877820.029922:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877820.029926:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877820.029933:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877820.029950:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511421824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877820.029962:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877820.029967:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877820.029969:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877820.029973:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2806 00000100:00080000:2.0:1788877820.029978:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877820.029982:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877820.029987:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877820.030003:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511421952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877820.030016:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877820.030020:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877820.030023:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877820.030027:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2806 00000100:00080000:2.0:1788877820.030030:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877820.030035:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877820.030042:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877820.030057:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511422080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877820.669662:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877821.693695:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877822.717635:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877823.741640:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877824.765631:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877825.149490:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877825.149499:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877825.149505:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877825.149516:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2811 00000100:00080000:2.0:1788877825.149548:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877825.149558:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877825.149606:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511422208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877825.149645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877825.149650:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877825.149652:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877825.149658:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2811 00000100:00080000:2.0:1788877825.149663:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877825.149669:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877825.149683:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877825.149718:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511422336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877825.149733:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877825.149736:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877825.149738:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877825.149741:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2811 00000100:00080000:2.0:1788877825.149745:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877825.149748:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877825.149754:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877825.149786:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511422464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877825.149808:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877825.149813:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877825.149815:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877825.149858:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2811 00000100:00080000:2.0:1788877825.149863:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877825.149872:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877825.149880:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877825.149894:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511422592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877825.149904:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877825.149908:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877825.149910:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877825.149916:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2811 00000100:00080000:2.0:1788877825.149919:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877825.149928:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877825.149937:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877825.149951:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511422720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877825.789593:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877826.813656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877827.837540:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877828.861518:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877829.885514:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877830.269488:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877830.269498:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877830.269503:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877830.269512:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2816 00000100:00080000:3.0:1788877830.269521:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877830.269527:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877830.269558:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511422848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877830.269580:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877830.269582:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877830.269584:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877830.269587:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2816 00000100:00080000:3.0:1788877830.269590:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877830.269593:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877830.269601:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877830.269613:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511422976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877830.269622:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877830.269624:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877830.269626:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877830.269628:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2816 00000100:00080000:3.0:1788877830.269631:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877830.269633:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877830.269636:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877830.269645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511423104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877830.269656:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877830.269659:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877830.269660:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877830.269663:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2816 00000100:00080000:3.0:1788877830.269665:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877830.269667:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877830.269670:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877830.269680:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511423232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877830.269686:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877830.269689:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877830.269691:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877830.269694:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2816 00000100:00080000:3.0:1788877830.269697:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877830.269699:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877830.269702:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877830.269710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511423360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877830.909678:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877831.933648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877832.957662:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877833.981640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877835.005557:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877835.389570:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877835.389584:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877835.389590:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877835.389604:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2822 00000100:00080000:2.0:1788877835.389621:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877835.389632:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877835.389706:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511423488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877835.389757:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877835.389762:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877835.389765:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877835.389771:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2822 00000100:00080000:2.0:1788877835.389802:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877835.389810:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877835.389825:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877835.389880:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511423616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877835.389909:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877835.389917:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877835.389921:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877835.389938:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2822 00000100:00080000:2.0:1788877835.389948:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877835.389956:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877835.389965:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877835.389986:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511423744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877835.390000:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877835.390007:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877835.390013:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877835.390025:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2822 00000100:00080000:2.0:1788877835.390033:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877835.390045:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877835.390066:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877835.390088:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511423872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877835.390117:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877835.390126:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877835.390134:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877835.390143:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2822 00000100:00080000:2.0:1788877835.390148:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877835.390153:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877835.390159:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877835.390178:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511424000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877836.029644:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877837.053550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877838.077583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877839.101525:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877840.125543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877840.509487:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877840.509495:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877840.509499:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877840.509508:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2827 00000100:00080000:2.0:1788877840.509518:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877840.509525:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877840.509559:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511424128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877840.509581:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877840.509584:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877840.509585:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877840.509589:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2827 00000100:00080000:2.0:1788877840.509591:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877840.509594:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877840.509602:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877840.509615:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511424256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877840.509627:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877840.509629:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877840.509631:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877840.509634:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2827 00000100:00080000:2.0:1788877840.509637:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877840.509639:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877840.509643:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877840.509651:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511424384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877840.509659:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877840.509664:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877840.509666:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877840.509670:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2827 00000100:00080000:2.0:1788877840.509674:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877840.509678:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877840.509683:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877840.509710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511424512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877840.509721:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877840.509724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877840.509727:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877840.509733:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2827 00000100:00080000:2.0:1788877840.509741:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877840.509747:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877840.509764:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877840.509778:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511424640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877841.149549:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877842.173658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877843.197586:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788877843.581983:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877843.582070:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877843.582098:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877843.582159:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877843.582180:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877843.582202:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877843.582237:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00010000:00010000:2.0:1788877844.221631:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877845.245704:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877845.629596:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877845.629617:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877845.629626:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877845.629648:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2832 00000100:00080000:3.0:1788877845.629671:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877845.629689:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877845.629806:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511424768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877845.629854:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877845.629860:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877845.629863:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877845.629869:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2832 00000100:00080000:3.0:1788877845.629875:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877845.629882:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877845.629900:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877845.629924:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511424896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877845.629945:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877845.629950:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877845.629952:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877845.629957:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2832 00000100:00080000:3.0:1788877845.629962:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877845.629966:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877845.629973:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877845.629994:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511425024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877845.630013:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877845.630021:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877845.630025:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877845.630032:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2832 00000100:00080000:3.0:1788877845.630043:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877845.630062:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877845.630075:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877845.630102:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511425152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877845.630123:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877845.630134:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877845.630141:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877845.630156:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2832 00000100:00080000:3.0:1788877845.630171:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877845.630184:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877845.630195:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877845.630226:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511425280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1788877846.269785:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877847.293585:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877848.317524:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877849.341604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877850.365669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877850.749596:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877850.749608:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877850.749614:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877850.749628:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2837 00000100:00080000:3.0:1788877850.749644:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877850.749655:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877850.749735:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511425408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877850.749774:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877850.749780:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877850.749783:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877850.749788:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2837 00000100:00080000:3.0:1788877850.749794:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877850.749800:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877850.749813:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877850.749835:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511425536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877850.749851:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877850.749856:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877850.749865:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877850.749874:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2837 00000100:00080000:3.0:1788877850.749885:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877850.749890:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877850.749896:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877850.749917:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511425664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877850.749930:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877850.749935:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877850.749937:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877850.749941:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2837 00000100:00080000:3.0:1788877850.749947:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877850.749952:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877850.749961:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877850.749990:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511425792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877850.750014:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877850.750026:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877850.750035:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877850.750054:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2837 00000100:00080000:3.0:1788877850.750067:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877850.750082:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877850.750103:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877850.750132:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511425920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877851.389667:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877852.413622:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877853.437662:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877854.461655:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877855.485653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877855.869667:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877855.869679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877855.869707:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877855.869721:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2842 00000100:00080000:1.0:1788877855.869735:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877855.869745:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877855.869796:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511426048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877855.869834:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877855.869839:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877855.869841:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877855.869847:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2842 00000100:00080000:1.0:1788877855.869851:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877855.869855:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877855.869867:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877855.869884:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511426176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877855.869899:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877855.869904:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877855.869907:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877855.869917:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2842 00000100:00080000:1.0:1788877855.869921:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877855.869934:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877855.869944:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877855.869960:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511426304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877855.869983:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877855.869988:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877855.869991:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877855.869995:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2842 00000100:00080000:1.0:1788877855.870019:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877855.870024:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877855.870031:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877855.870055:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511426432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877855.870070:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877855.870075:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877855.870077:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877855.870084:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2842 00000100:00080000:1.0:1788877855.870090:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877855.870095:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877855.870103:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877855.870124:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511426560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877856.509728:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877857.533605:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877858.557520:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877859.581735:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877860.605654:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877860.989502:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877860.989518:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877860.989527:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877860.989546:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2847 00000100:00080000:1.0:1788877860.989568:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877860.989583:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877860.989637:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511426688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877860.989718:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877860.989727:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877860.989730:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877860.989739:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2847 00000100:00080000:1.0:1788877860.989748:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877860.989756:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877860.989770:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877860.989798:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511426816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877860.989826:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877860.989834:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877860.989839:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877860.989847:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2847 00000100:00080000:1.0:1788877860.989872:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877860.989886:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877860.989904:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877860.989936:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511426944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877860.990002:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877860.990012:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877860.990020:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877860.990038:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2847 00000100:00080000:1.0:1788877860.990048:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877860.990061:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877860.990077:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877860.990101:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511427072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877860.990135:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877860.990141:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877860.990145:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877860.990153:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2847 00000100:00080000:1.0:1788877860.990163:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877860.990176:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877860.990184:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877860.990205:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511427200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1788877861.629655:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877862.653645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877863.677706:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877864.701828:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877865.725611:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877866.109544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877866.109554:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877866.109559:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877866.109573:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2852 00000100:00080000:1.0:1788877866.109588:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877866.109598:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877866.109656:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511427328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877866.109686:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877866.109691:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877866.109694:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877866.109700:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2852 00000100:00080000:1.0:1788877866.109705:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877866.109710:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877866.109719:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877866.109722:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511427328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877866.109736:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511427456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877866.109744:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877866.109746:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877866.109748:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877866.109750:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2852 00000100:00080000:1.0:1788877866.109752:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877866.109755:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877866.109758:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877866.109766:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511427584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877866.109772:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877866.109775:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877866.109776:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877866.109778:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2852 00000100:00080000:1.0:1788877866.109781:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788877866.109785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877866.109787:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877866.109792:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877866.109800:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511427712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877866.109806:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877866.109808:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877866.109810:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877866.109813:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2852 00000100:00080000:1.0:1788877866.109816:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877866.109819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877866.109822:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877866.109823:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877866/real 1788877866] req@ffffa06188699f80 x1875771511427328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877866.109830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511427840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877866.109848:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511427328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877866.109857:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877866.109861:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877866.109865:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511427328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877866.109884:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877866.109898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877866.109906:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877866/real 1788877866] req@ffffa06188698000 x1875771511427456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877866.109914:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511427456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877866.109920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877866.109922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877866.109924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877866.109927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877866.109938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877866.109943:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877866/real 1788877866] req@ffffa0618869ad80 x1875771511427584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877866.109950:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511427584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877866.109955:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877866.109957:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877866.109959:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877866.109963:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877866.109974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877866.109979:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877866/real 1788877866] req@ffffa0618869bb80 x1875771511427712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877866.109986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511427712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877866.109992:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877866.109993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877866.109995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877866.109998:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877866.110007:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877866.110013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877866/real 1788877866] req@ffffa0618869a680 x1875771511427840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877866.110019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511427840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877866.110024:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877866.110026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877866.110028:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511427840:192.168.203.111@tcp:8:kworker.0 00010000:00010000:2.0:1788877866.749749:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877867.773588:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877868.797692:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877869.821729:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877870.845552:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877871.229585:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877871.229598:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877871.229604:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877871.229652:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2857 00000100:00080000:1.0:1788877871.229663:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877871.229670:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877871.229708:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511427968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877871.229762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877871.229770:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877871.229773:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877871.229779:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2857 00000100:00080000:1.0:1788877871.229784:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877871.229789:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877871.229801:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877871.229819:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511428096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877871.229834:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877871.229843:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1788877871.229844:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511427968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877871.229846:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877871.229862:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2857 00000100:00080000:1.0:1788877871.229867:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877871.229872:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877871.229879:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877871.229895:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511428224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877871.229907:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877871.229912:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877871.229915:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877871.229919:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2857 00000100:00080000:1.0:1788877871.229935:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788877871.229946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877871.229947:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877871.229978:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877871.229990:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511428352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877871.229999:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877871.230002:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877871.230004:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877871.230008:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2857 00000100:00080000:1.0:1788877871.230012:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877871.230016:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877871/real 1788877871] req@ffffa0618869b800 x1875771511427968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877871.230020:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877871.230029:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877871.230042:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511428480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877871.230048:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511427968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877871.230065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877871.230069:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877871.230091:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511427968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877871.230124:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877871.230156:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877871.230189:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877871/real 1788877871] req@ffffa0618869aa00 x1875771511428096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877871.230207:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511428096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877871.230218:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877871.230225:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877871.230232:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877871.230244:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877871.230284:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877871.230308:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877871/real 1788877871] req@ffffa0618869b100 x1875771511428224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877871.230334:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511428224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877871.230350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877871.230357:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877871.230364:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877871.230376:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877871.230478:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877871.230490:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877871/real 1788877871] req@ffffa06188698380 x1875771511428352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877871.230501:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511428352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877871.230509:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877871.230511:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877871.230514:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877871.230519:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877871.230534:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877871.230542:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877871/real 1788877871] req@ffffa06188699c00 x1875771511428480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877871.230551:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511428480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877871.230563:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877871.230566:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877871.230586:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428480:192.168.203.111@tcp:8:kworker.0 00010000:00010000:2.0:1788877871.869759:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877872.893580:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877873.917568:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877874.941547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877875.965561:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877876.349458:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877876.349467:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877876.349471:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877876.349480:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2863 00000100:00080000:1.0:1788877876.349491:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877876.349497:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877876.349529:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511428608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877876.349551:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877876.349555:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877876.349557:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877876.349562:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2863 00000100:00080000:1.0:1788877876.349566:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877876.349571:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877876.349581:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877876.349627:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511428736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877876.349632:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511428608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877876.349646:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877876.349652:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877876.349655:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877876.349660:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2863 00000100:00080000:1.0:1788877876.349666:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877876.349673:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877876.349680:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877876.349696:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511428864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877876.349713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877876.349727:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877876.349732:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877876.349736:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877876.349742:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2863 00000100:00080000:1.0:1788877876.349747:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877876.349753:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877876.349759:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877876/real 1788877876] req@ffffa06188699c00 x1875771511428608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788877876.349761:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877876.349777:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511428992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877876.349787:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511428608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877876.349804:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877876.349811:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788877876.349812:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877876.349818:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1788877876.349818:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511428608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877876.349821:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877876.349827:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2863 00000100:00080000:1.0:1788877876.349832:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877876.349837:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877876.349843:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877876.349852:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788877876.349876:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511429120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877876.349893:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877876.349906:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877876/real 1788877876] req@ffffa06188698380 x1875771511428736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877876.349920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511428736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877876.349930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877876.349933:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877876.349936:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877876.349941:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877876.349956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877876.349964:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877876/real 1788877876] req@ffffa0618869b100 x1875771511428864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877876.349971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511428864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877876.349977:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877876.349979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877876.349981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877876.349984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877876.349995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877876.350000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877876/real 1788877876] req@ffffa0618869aa00 x1875771511428992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877876.350007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511428992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877876.350013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877876.350014:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877876.350017:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511428992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877876.350028:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877876.350037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877876.350043:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877876/real 1788877876] req@ffffa0618869b800 x1875771511429120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877876.350049:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511429120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877876.350055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877876.350057:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877876.350058:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429120:192.168.203.111@tcp:8:kworker.0 00010000:00010000:2.0:1788877876.989547:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877878.013653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877879.037683:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877880.061672:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877881.085617:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877881.469451:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877881.469460:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877881.469464:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877881.469473:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2868 00000100:00080000:1.0:1788877881.469482:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877881.469489:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877881.469521:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511429248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877881.469543:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877881.469546:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877881.469548:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877881.469551:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2868 00000100:00080000:1.0:1788877881.469553:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877881.469555:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877881.469564:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877881.469587:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511429376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877881.469596:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877881.469599:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877881.469600:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877881.469605:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2868 00000100:00100000:2.0:1788877881.469606:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511429248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877881.469608:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877881.469615:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877881.469619:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877881.469629:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511429504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877881.469635:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877881.469638:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877881.469639:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877881.469642:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2868 00000100:00080000:1.0:1788877881.469644:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877881.469657:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877881.469661:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877881.469667:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511429632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877881.469674:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877881.469676:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877881.469678:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788877881.469681:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877881.469688:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2868 00000100:00080000:1.0:1788877881.469694:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877881.469698:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877881.469704:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877881.469724:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877881/real 1788877881] req@ffffa0618869a680 x1875771511429248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877881.469731:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511429760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877881.469743:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511429248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877881.469753:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877881.469756:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877881.469760:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511429248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877881.469777:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877881.469790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877881.469798:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877881/real 1788877881] req@ffffa0618869bb80 x1875771511429376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877881.469817:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511429376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877881.469822:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877881.469825:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877881.469827:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877881.469830:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877881.469840:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877881.469846:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877881/real 1788877881] req@ffffa0618869ad80 x1875771511429504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877881.469853:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511429504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877881.469858:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877881.469860:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877881.469862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877881.469866:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877881.469886:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877881.469892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877881/real 1788877881] req@ffffa06188698000 x1875771511429632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877881.469899:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511429632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877881.469904:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877881.469906:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877881.469908:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877881.469911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877881.469920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877881.469925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877881/real 1788877881] req@ffffa06188699f80 x1875771511429760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877881.469932:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511429760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877881.469937:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877881.469939:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877881.469940:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511429760:192.168.203.111@tcp:8:kworker.0 00010000:00010000:2.0:1788877882.109504:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877883.133517:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877884.157659:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877885.181641:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877886.205719:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788877886.589536:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877886.589561:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877886.589565:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877886.589574:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2873 00000100:00080000:3.0:1788877886.589603:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877886.589612:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877886.589658:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511429888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877886.589724:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877886.589734:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877886.589738:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877886.589745:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2873 00000100:00080000:3.0:1788877886.589751:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877886.589758:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877886.589771:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877886.589808:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511430016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877886.589812:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511429888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877886.589830:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877886.589835:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877886.589839:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877886.589845:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2873 00000100:00080000:3.0:1788877886.589851:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877886.589874:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877886.589882:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877886.589897:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511430144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877886.589911:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877886.589917:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1788877886.589918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877886.589936:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877886.589940:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2873 00000100:00080000:3.0:1788877886.589943:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877886.589947:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877886.589952:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877886.589962:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511430272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877886.589968:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877886/real 1788877886] req@ffffa06188872300 x1875771511429888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877886.589972:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877886.589976:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877886.589978:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877886.589982:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2873 00000100:00080000:3.0:1788877886.589985:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877886.590005:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511429888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877886.590006:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877886.590013:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1788877886.590021:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877886.590027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877886.590027:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511430400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877886.590034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511429888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877886.590077:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877886.590114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877886.590142:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877886/real 1788877886] req@ffffa06188873b80 x1875771511430016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877886.590155:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511430016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877886.590164:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877886.590167:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877886.590171:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877886.590177:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877886.590196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877886.590205:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877886/real 1788877886] req@ffffa06188870700 x1875771511430144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877886.590216:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511430144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877886.590245:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877886.590247:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877886.590250:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877886.590254:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877886.590266:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877886.590272:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877886/real 1788877886] req@ffffa06188872680 x1875771511430272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877886.590280:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511430272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877886.590286:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877886.590288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877886.590308:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877886.590315:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877886.590334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877886.590345:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877886/real 1788877886] req@ffffa06188872d80 x1875771511430400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877886.590358:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511430400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877886.590367:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877886.590371:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877886.590375:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430400:192.168.203.111@tcp:8:kworker.0 00010000:00010000:2.0:1788877887.229697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877888.253695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877889.277750:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877890.301566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877891.325577:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788877891.709494:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877891.709503:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877891.709507:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877891.709516:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2878 00000100:00080000:1.0:1788877891.709537:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877891.709545:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877891.709590:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511430528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877891.709612:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877891.709614:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877891.709616:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877891.709619:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2878 00000100:00080000:1.0:1788877891.709622:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877891.709624:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877891.709632:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877891.709642:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511430656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877891.709651:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877891.709654:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877891.709655:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877891.709661:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2878 00000100:00100000:2.0:1788877891.709661:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511430528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877891.709666:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877891.709678:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877891.709682:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877891.709694:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511430784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877891.709701:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877891.709704:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877891.709706:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877891.709709:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2878 00000100:00080000:1.0:1788877891.709711:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877891.709714:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877891.709717:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877891.709726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511430912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788877891.709736:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877891.709740:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877891.709745:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877891.709747:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877891.709761:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2878 00000100:00080000:1.0:1788877891.709767:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877891.709772:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877891.709778:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877891.709792:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877891/real 1788877891] req@ffffa06188699f80 x1875771511430528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877891.709793:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511431040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877891.709838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511430528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877891.709848:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877891.709851:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877891.709855:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511430528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877891.709873:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877891.709891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877891.709900:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877891/real 1788877891] req@ffffa06188698000 x1875771511430656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877891.709908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511430656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877891.709914:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877891.709916:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877891.709918:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877891.709923:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877891.709948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877891.709955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877891/real 1788877891] req@ffffa0618869ad80 x1875771511430784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877891.709963:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511430784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877891.709968:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877891.709970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877891.709972:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877891.709976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877891.709986:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877891.709992:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877891/real 1788877891] req@ffffa0618869bb80 x1875771511430912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877891.709999:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511430912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877891.710004:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877891.710006:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877891.710008:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511430912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877891.710011:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877891.710021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877891.710026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877891/real 1788877891] req@ffffa0618869a680 x1875771511431040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877891.710033:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511431040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877891.710038:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877891.710040:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877891.710042:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431040:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788877892.349626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877893.373675:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877894.397564:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1788877895.421561:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1788877896.445646:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788877896.829636:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877896.829651:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877896.829658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877896.829673:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2883 00000100:00080000:2.0:1788877896.829688:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877896.829699:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877896.829748:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511431168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877896.829794:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877896.829801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877896.829806:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877896.829814:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2883 00000100:00080000:2.0:1788877896.829822:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877896.829830:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877896.829845:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877896.829884:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511431296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877896.829905:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877896.829911:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877896.829915:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877896.829921:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2883 00000100:00080000:2.0:1788877896.829927:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877896.829933:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877896.829943:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877896.829962:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511431424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877896.829981:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877896.829987:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877896.829990:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877896.829995:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2883 00000100:00080000:2.0:1788877896.830001:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877896.830007:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877896.830015:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877896.830032:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511431552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877896.830056:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877896.830061:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877896.830065:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877896.830073:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2883 00000100:00080000:2.0:1788877896.830079:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877896.830085:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877896.830094:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877896.830111:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511431680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877896.830346:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877896.830354:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877896.830373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877896.830427:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877896/real 1788877896] req@ffffa061888aed80 x1875771511431552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877896.830442:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511431552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877896.830452:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877896.830456:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877896.830460:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877896.830467:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877896.830489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877896.830531:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877896/real 1788877896] req@ffffa061888ad880 x1875771511431680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877896.830552:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511431680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877896.830574:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877896.830578:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877896.830582:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877897.469557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877897.469676:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877897.725561:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877898.493449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877898.493540:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877898.749459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877899.517514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877899.517606:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877899.773559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877900.541552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877900.541691:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877900.797582:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877901.565512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877901.565648:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877901.821594:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877901.949507:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877901.949519:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877901.949535:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877901.949549:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2888 00000100:00080000:2.0:1788877901.949564:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877901.949574:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877901.949623:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511431808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877901.949669:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877901.949674:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877901.949677:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877901.949683:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2888 00000100:00080000:2.0:1788877901.949689:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877901.949696:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877901.949711:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788877901.949714:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511431808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877901.949735:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511431936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877901.949752:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877901.949757:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877901.949768:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877901.949773:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2888 00000100:00080000:2.0:1788877901.949779:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877901.949786:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877901.949794:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788877901.949795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877901.949817:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511432064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877901.949843:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788877901.949844:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877901/real 1788877901] req@ffffa061888aea00 x1875771511431808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877901.949857:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877901.949863:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788877901.949875:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511431808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877901.949875:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2888 00000100:00080000:2.0:1788877901.949885:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788877901.949891:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877901.949895:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788877901.949901:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877901.949907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511431808:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788877901.949908:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877901.949930:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511432192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877901.949934:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431936:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788877901.949969:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788877901.949976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877901.949977:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877901.949983:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877901.949992:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2888 00000100:00100000:0.0:1788877901.949996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877901/real 1788877901] req@ffffa061888ac380 x1875771511431936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877901.950002:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877901.950015:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788877901.950017:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511431936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788877901.950026:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788877901.950029:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877901.950034:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877901.950038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511431936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877901.950051:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877901.950052:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511432320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788877901.950073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877901.950086:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877901/real 1788877901] req@ffffa061888aca80 x1875771511432064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877901.950097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511432064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877901.950105:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877901.950108:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877901.950111:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877901.950121:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877901.950134:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877901.950142:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877901/real 1788877901] req@ffffa061888adc00 x1875771511432192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877901.950151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511432192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877901.950159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877901.950161:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877901.950164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877901.950169:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877901.950181:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877901.950189:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877901/real 1788877901] req@ffffa061888af100 x1875771511432320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877901.950199:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511432320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877901.950206:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877901.950208:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877901.950211:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877902.589469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877902.589536:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877902.845464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877903.613444:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877903.613534:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877903.869592:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877904.637557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877904.637657:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877904.893618:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877905.661494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877905.661645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877905.917596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877906.685628:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877906.685693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877906.941559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877907.069651:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877907.069687:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877907.069692:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877907.069704:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2893 00000100:00080000:3.0:1788877907.069717:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877907.069737:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877907.069783:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511432448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877907.069815:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877907.069820:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877907.069822:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877907.069827:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2893 00000100:00080000:3.0:1788877907.069832:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877907.069837:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877907.069849:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877907.069866:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511432576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877907.069887:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877907.069892:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877907.069896:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788877907.069901:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511432448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877907.069903:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2893 00000100:00080000:3.0:1788877907.069918:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877907.069923:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877907.069930:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877907.069957:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511432704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877907.069970:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877907.069973:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877907.069975:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877907.069980:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2893 00000100:00080000:3.0:1788877907.069984:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877907.069987:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877907.069993:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877907.070005:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511432832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877907.070023:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877907.070027:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877907.070030:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877907.070035:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2893 00000100:00080000:3.0:1788877907.070040:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877907.070055:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877907.070062:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877907.070063:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877907.070083:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511432960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877907.070131:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877907.070159:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877907.070187:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877907.709462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877907.709536:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877907.965513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877908.733556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877908.733645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877908.989524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877909.757477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877909.757528:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877910.013559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877910.141720:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877910.141771:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877910.141789:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877910.141828:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877910.141837:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877910.141856:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877910.141873:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788877910.141895:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877907/real 1788877910] req@ffffa06188871880 x1875771511432576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877910.141929:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511432576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877910.141942:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877910.141947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877910.141951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877910.141960:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877907/real 1788877910] req@ffffa06188870a80 x1875771511432704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877910.141974:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511432704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877910.141982:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877910.141985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877910.141988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877910.141995:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877907/real 1788877910] req@ffffa06188870380 x1875771511432832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877910.142006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511432832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877910.142013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877910.142017:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877910.142020:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877910.142027:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877907/real 1788877910] req@ffffa06188871c00 x1875771511432448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877910.142037:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511432448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877910.142044:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877910.142054:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877910.142058:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511432448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877910.142090:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877907/real 1788877910] req@ffffa06188872d80 x1875771511432960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877910.142104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511432960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877910.142111:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877910.142115:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877910.142118:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511432960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877910.781565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877910.781702:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877911.037631:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877911.805607:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877911.805670:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877912.061558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877912.189611:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877912.189621:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877912.189625:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877912.189635:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2898 00000100:00080000:1.0:1788877912.189646:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877912.189654:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877912.189690:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511433088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877912.189719:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877912.189723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877912.189725:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877912.189730:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2898 00000100:00080000:1.0:1788877912.189734:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877912.189738:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877912.189749:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877912.189760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511433216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877912.189766:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511433088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877912.189780:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877912.189784:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877912.189786:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877912.189790:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2898 00000100:00080000:1.0:1788877912.189795:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877912.189801:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877912.189808:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877912.189829:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511433344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788877912.189844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877912.189846:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877912.189853:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877912.189856:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877912.189864:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2898 00000100:00080000:1.0:1788877912.189872:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877912.189879:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788877912.189885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877912/real 1788877912] req@ffffa0618869b800 x1875771511433088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788877912.189890:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877912.189913:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511433088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877912.189924:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877912.189929:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788877912.189936:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511433472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877912.189937:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511433088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877912.189959:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877912.189981:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877912.189984:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788877912.189989:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433216:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788877912.189993:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2898 00000100:00080000:1.0:1788877912.190003:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877912.190010:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877912.190019:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877912.190053:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511433600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788877912.190066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877912.190092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877912/real 1788877912] req@ffffa0618869aa00 x1875771511433216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877912.190122:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511433216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877912.190135:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877912.190140:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877912.190145:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877912.190153:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877912.190190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877912.190209:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877912/real 1788877912] req@ffffa0618869b100 x1875771511433344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877912.190227:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511433344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877912.190247:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877912.190252:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877912.190257:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877912.190266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877912.190296:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877912.190319:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877912/real 1788877912] req@ffffa06188698380 x1875771511433472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877912.190335:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511433472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877912.190347:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877912.190351:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877912.190356:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877912.190379:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877912.190457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877912.190474:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877912/real 1788877912] req@ffffa06188699c00 x1875771511433600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877912.190489:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511433600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877912.190500:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877912.190505:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877912.190510:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877912.829548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877912.829649:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877913.085612:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877913.853532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877913.853604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877914.109526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877914.877582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877914.877666:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877915.133518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877915.901540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877915.901659:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877916.157518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877916.925514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877916.925590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877917.181541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877917.309497:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877917.309525:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877917.309532:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877917.309543:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2904 00000100:00080000:1.0:1788877917.309555:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877917.309563:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877917.309619:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511433728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877917.309673:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877917.309677:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877917.309679:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877917.309683:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2904 00000100:00080000:1.0:1788877917.309687:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877917.309691:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877917.309703:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877917.309730:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511433856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877917.309739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877917.309742:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788877917.309743:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511433728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877917.309743:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877917.309748:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2904 00000100:00080000:1.0:1788877917.309752:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877917.309756:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877917.309759:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877917.309767:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511433984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877917.309773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877917.309776:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877917.309777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877917.309779:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2904 00000100:00080000:1.0:1788877917.309782:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877917.309785:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877917.309789:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877917.309796:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511434112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877917.309802:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877917.309804:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877917.309806:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877917.309808:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2904 00000800:00000100:0.0:1788877917.309813:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877917.309814:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877917.309819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877917.309823:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877917.309830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511434240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877917.309850:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877917/real 1788877917] req@ffffa06188699c00 x1875771511433728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877917.309864:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511433728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877917.309872:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877917.309875:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877917.309878:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511433728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877917.309895:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877917.309910:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877917.309918:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877917/real 1788877917] req@ffffa06188698380 x1875771511433856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877917.309926:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511433856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877917.309933:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877917.309935:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877917.309937:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877917.309940:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877917.309950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877917.309956:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877917/real 1788877917] req@ffffa0618869b100 x1875771511433984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877917.309963:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511433984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877917.309968:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877917.309970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877917.309972:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511433984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877917.309976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877917.309986:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877917.309992:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877917/real 1788877917] req@ffffa0618869aa00 x1875771511434112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877917.310019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511434112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877917.310027:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877917.310029:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877917.310033:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877917.310039:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877917.310054:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877917.310062:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877917/real 1788877917] req@ffffa0618869b800 x1875771511434240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877917.310072:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511434240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877917.310080:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877917.310082:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877917.310085:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877917.949467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877917.949577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877918.205473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877918.973584:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877918.973654:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877919.229501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877919.997466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877919.997518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877920.253524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877921.021501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877921.021607:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877921.277497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877922.045475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877922.045543:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877922.301578:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877922.429544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877922.429558:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877922.429565:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877922.429581:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2909 00000100:00080000:1.0:1788877922.429597:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877922.429607:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877922.429657:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511434368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877922.429697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877922.429703:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877922.429705:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877922.429711:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2909 00000100:00080000:1.0:1788877922.429716:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877922.429722:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877922.429735:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877922.429756:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511434496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877922.429759:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511434368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877922.429787:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877922.429792:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877922.429795:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877922.429800:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2909 00000100:00080000:1.0:1788877922.429805:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877922.429810:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877922.429817:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877922.429833:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511434624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877922.429846:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877922.429850:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877922.429853:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788877922.429862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877922.429865:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2909 00000100:00080000:1.0:1788877922.429873:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877922.429879:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877922.429887:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877922.429906:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511434752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877922.429920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877922/real 1788877922] req@ffffa0618869a680 x1875771511434368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877922.429921:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877922.429936:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877922.429941:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877922.429956:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2909 00000100:00080000:1.0:1788877922.429964:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877922.429975:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877922.429987:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877922.430005:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511434880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877922.430015:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511434368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877922.430041:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877922.430050:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877922.430061:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511434368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877922.430103:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877922.430139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877922.430157:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877922/real 1788877922] req@ffffa0618869bb80 x1875771511434496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877922.430174:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511434496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877922.430186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877922.430189:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877922.430194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877922.430202:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877922.430222:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877922.430234:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877922/real 1788877922] req@ffffa0618869ad80 x1875771511434624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877922.430248:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511434624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877922.430267:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877922.430271:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877922.430275:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877922.430282:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877922.430302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877922.430313:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877922/real 1788877922] req@ffffa06188698000 x1875771511434752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877922.430327:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511434752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877922.430337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877922.430340:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877922.430344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877922.430351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877922.430370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877922.430430:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877922/real 1788877922] req@ffffa06188699f80 x1875771511434880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877922.430450:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511434880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877922.430461:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877922.430465:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877922.430470:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511434880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877923.069540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877923.069653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877923.325634:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877924.093486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877924.093534:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877924.349487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:2.0:1788877925.117593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877925.117609:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788877925.373615:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877926.141661:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877926.141676:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877926.397501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877927.165546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877927.165670:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877927.421525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877927.549544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877927.549559:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877927.549566:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877927.549580:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2914 00000100:00080000:1.0:1788877927.549595:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877927.549605:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877927.549651:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511435008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877927.549687:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877927.549692:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877927.549695:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877927.549700:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2914 00000100:00080000:1.0:1788877927.549705:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877927.549709:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877927.549721:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877927.549735:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511435136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877927.549747:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877927.549751:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877927.549753:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877927.549756:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2914 00000100:00080000:1.0:1788877927.549759:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877927.549763:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877927.549768:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877927.549780:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511435264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877927.549794:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877927.549799:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877927.549803:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788877927.549808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511435008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877927.549812:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2914 00000100:00080000:1.0:1788877927.549826:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877927.549830:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877927.549836:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877927.549855:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511435392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877927.549867:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877927.549872:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877927.549874:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877927.549878:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2914 00000100:00080000:1.0:1788877927.549885:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877927.549889:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877927.549895:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788877927.549902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877927.549915:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511435520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877927.549975:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877927/real 1788877927] req@ffffa06188699f80 x1875771511435008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877927.549999:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511435008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877927.550011:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877927.550015:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877927.550020:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511435008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877927.550043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877927.550070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877927.550084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877927/real 1788877927] req@ffffa06188698000 x1875771511435136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877927.550098:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511435136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877927.550108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877927.550112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877927.550116:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877927.550142:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877927.550161:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877927.550172:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877927/real 1788877927] req@ffffa0618869ad80 x1875771511435264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877927.550184:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511435264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877927.550194:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877927.550198:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877927.550201:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877927.550207:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877927.550223:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877927.550233:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877927/real 1788877927] req@ffffa0618869bb80 x1875771511435392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877927.550245:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511435392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877927.550254:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877927.550257:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877927.550261:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877927.550267:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877927.550283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877927.550293:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877927/real 1788877927] req@ffffa0618869a680 x1875771511435520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877927.550306:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511435520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877927.550321:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877927.550326:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877927.550332:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877928.189518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877928.189593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877928.445595:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877929.213521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877929.213697:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877929.469549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877930.237545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877930.237666:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877930.493558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877931.261459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877931.261575:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877931.517526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877932.285522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877932.285676:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877932.541548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877932.669552:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877932.669565:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877932.669571:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877932.669587:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2919 00000100:00080000:2.0:1788877932.669600:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877932.669626:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877932.669670:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511435648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877932.669700:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877932.669703:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877932.669705:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877932.669710:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2919 00000100:00080000:2.0:1788877932.669713:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877932.669717:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877932.669727:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877932.669748:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511435776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877932.669749:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511435648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877932.669782:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877932.669785:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877932.669787:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877932.669791:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2919 00000100:00080000:2.0:1788877932.669794:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877932.669798:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877932.669803:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877932.669816:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511435904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877932.669843:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877932.669847:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877932.669849:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788877932.669855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877932.669859:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2919 00000100:00080000:2.0:1788877932.669865:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877932.669869:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877932.669891:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877932.669922:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511436032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877932.669934:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788877932.669942:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877932/real 1788877932] req@ffffa061888afb80 x1875771511435648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877932.669942:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877932.669945:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877932.669955:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2919 00000100:00080000:2.0:1788877932.669993:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877932.670001:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877932.670010:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788877932.670027:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511436160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877932.670041:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511435648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877932.670055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877932.670060:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877932.670065:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511435648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877932.670089:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877932.670116:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877932.670129:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877932/real 1788877932] req@ffffa061888ae680 x1875771511435776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877932.670141:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511435776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877932.670149:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877932.670152:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877932.670156:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877932.670161:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877932.670197:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877932.670210:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877932/real 1788877932] req@ffffa061888ae300 x1875771511435904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877932.670225:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511435904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877932.670243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877932.670247:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877932.670251:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511435904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877932.670258:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877932.670288:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877932.670308:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877932/real 1788877932] req@ffffa061888af480 x1875771511436032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877932.670367:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511436032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877932.670410:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877932.670415:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877932.670418:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877932.670425:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877932.670450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877932.670463:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877932/real 1788877932] req@ffffa061888ac000 x1875771511436160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877932.670474:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511436160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877932.670482:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877932.670485:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877932.670488:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877933.309470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877933.309523:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877933.565491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877934.333458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877934.333519:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877934.589459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877935.357479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877935.357546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877935.613552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877936.381479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877936.381531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877936.637500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877937.405589:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877937.405682:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877937.661517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877937.789583:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877937.789602:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877937.789609:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877937.789626:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2924 00000100:00080000:3.0:1788877937.789640:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877937.789650:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877937.789710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511436288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877937.789752:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877937.789758:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877937.789760:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877937.789777:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2924 00000100:00080000:3.0:1788877937.789783:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877937.789791:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877937.789809:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877937.789839:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511436416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877937.789848:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511436288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877937.789882:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877937.789888:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877937.789892:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877937.789898:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2924 00000100:00080000:3.0:1788877937.789905:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877937.789912:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877937.789921:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877937.789942:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511436544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788877937.789954:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877937.789969:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877937.789974:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877937.789976:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877937.789984:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2924 00000100:00080000:3.0:1788877937.789991:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877937.789996:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877937.790004:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877937.790020:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877937/real 1788877937] req@ffffa06188872680 x1875771511436288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788877937.790022:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511436672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877937.790086:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877937.790092:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877937.790105:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877937.790115:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2924 00000100:00100000:0.0:1788877937.790117:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511436288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877937.790120:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877937.790131:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877937.790143:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788877937.790155:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877937.790166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788877937.790173:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511436800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877937.790178:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511436288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877937.790224:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877937.790282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877937.790309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877937/real 1788877937] req@ffffa06188870700 x1875771511436416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877937.790367:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511436416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877937.790411:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877937.790417:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877937.790422:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877937.790431:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877937.790464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877937.790482:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877937/real 1788877937] req@ffffa06188873b80 x1875771511436544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877937.790498:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511436544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877937.790509:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877937.790513:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877937.790517:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877937.790524:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877937.790545:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877937.790557:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877937/real 1788877937] req@ffffa06188872300 x1875771511436672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877937.790590:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511436672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877937.790607:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877937.790613:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877937.790620:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877937.790632:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877937.790672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877937.790707:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877937/real 1788877937] req@ffffa06188873480 x1875771511436800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877937.790725:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511436800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877937.790736:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877937.790740:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877937.790744:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511436800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877938.429501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877938.429567:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877938.685462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877939.453466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877939.453548:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877939.709502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877940.477516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877940.477695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877940.733464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877941.501464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877941.501536:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877941.757525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877942.525542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877942.525649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877942.781542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877942.909492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877942.909501:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877942.909505:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877942.909515:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2929 00000100:00080000:1.0:1788877942.909526:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877942.909534:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877942.909568:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511436928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877942.909595:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877942.909598:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877942.909599:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877942.909602:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2929 00000100:00080000:1.0:1788877942.909604:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877942.909608:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877942.909617:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877942.909629:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511437056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877942.909631:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511436928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877942.909646:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877942.909649:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877942.909650:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877942.909652:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2929 00000100:00080000:1.0:1788877942.909655:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877942.909658:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877942.909661:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877942.909671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511437184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877942.909677:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877942.909679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877942.909680:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877942.909683:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2929 00000100:00080000:1.0:1788877942.909687:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877942.909689:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877942.909693:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788877942.909697:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877942.909703:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511437312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877942.909710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877942.909713:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877942.909715:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877942.909718:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2929 00000100:00080000:1.0:1788877942.909721:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877942.909723:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877942.909727:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877942.909733:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877942/real 1788877942] req@ffffa0618869b800 x1875771511436928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788877942.909737:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511437440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877942.909775:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511436928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877942.909785:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877942.909789:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877942.909792:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511436928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877942.909827:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877942.909850:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877942.909878:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877942/real 1788877942] req@ffffa0618869aa00 x1875771511437056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877942.909890:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511437056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877942.909915:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877942.909918:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877942.909921:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877942.909927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877942.909945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877942.909954:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877942/real 1788877942] req@ffffa0618869b100 x1875771511437184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877942.909964:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511437184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877942.909977:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877942.909980:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877942.909983:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877942.909989:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877942.910002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877942.910010:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877942/real 1788877942] req@ffffa06188698380 x1875771511437312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877942.910035:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511437312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877942.910041:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877942.910043:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877942.910045:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877942.910049:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877942.910060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877942.910066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877942/real 1788877942] req@ffffa06188699c00 x1875771511437440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877942.910073:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511437440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877942.910079:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877942.910098:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877942.910101:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877943.549510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877943.549614:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877943.805512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877944.573511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877944.573613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877944.829524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788877945.597619:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877945.597683:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788877945.853455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877946.621459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877946.621519:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877946.877470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877947.645475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877947.645539:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877947.901466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877948.029466:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877948.029476:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877948.029480:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877948.029489:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2934 00000100:00080000:1.0:1788877948.029498:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877948.029506:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877948.029538:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511437568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877948.029562:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877948.029564:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877948.029566:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877948.029569:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2934 00000100:00080000:1.0:1788877948.029571:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877948.029574:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877948.029581:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877948.029589:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511437696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877948.029597:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877948.029600:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877948.029601:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877948.029604:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2934 00000100:00080000:1.0:1788877948.029606:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877948.029608:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877948.029612:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788877948.029618:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511437568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877948.029622:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511437824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877948.029633:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877948.029635:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877948.029636:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877948.029638:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2934 00000100:00080000:1.0:1788877948.029642:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877948.029644:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877948.029647:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877948.029655:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511437952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877948.029662:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877948.029665:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877948.029666:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877948.029668:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2934 00000100:00080000:1.0:1788877948.029680:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788877948.029686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788877948.029686:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877948.029693:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877948.029702:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511438080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877948.029727:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877948/real 1788877948] req@ffffa06188699c00 x1875771511437568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877948.029742:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511437568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877948.029749:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877948.029752:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877948.029755:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511437568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877948.029760:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877948.029776:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877948.029784:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877948/real 1788877948] req@ffffa06188698380 x1875771511437696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877948.029792:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511437696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877948.029798:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877948.029801:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877948.029803:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877948.029821:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877948.029832:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877948.029838:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877948/real 1788877948] req@ffffa0618869b100 x1875771511437824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877948.029845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511437824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877948.029851:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877948.029853:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877948.029854:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877948.029858:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877948.029869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877948.029878:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877948/real 1788877948] req@ffffa0618869aa00 x1875771511437952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877948.029892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511437952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877948.029900:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877948.029904:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877948.029907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511437952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877948.029913:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877948.029934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877948.029947:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877948/real 1788877948] req@ffffa0618869b800 x1875771511438080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877948.029959:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511438080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877948.029967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877948.029971:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877948.029974:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877948.669448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877948.669499:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877948.925472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877949.693450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877949.693500:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877949.949480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877950.717435:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877950.717501:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877950.973464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877951.741449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877951.741526:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877951.997486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877952.765442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877952.765539:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877953.021493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877953.149485:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877953.149493:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877953.149499:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877953.149508:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2939 00000100:00080000:1.0:1788877953.149524:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877953.149530:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877953.149562:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511438208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877953.149583:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877953.149586:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877953.149588:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877953.149591:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2939 00000100:00080000:1.0:1788877953.149593:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877953.149596:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877953.149604:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877953.149613:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511438336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877953.149621:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877953.149627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788877953.149629:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511438208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877953.149629:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877953.149644:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2939 00000100:00080000:1.0:1788877953.149646:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877953.149649:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877953.149652:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877953.149662:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511438464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877953.149668:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877953.149671:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877953.149673:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877953.149676:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2939 00000100:00080000:1.0:1788877953.149680:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877953.149683:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877953.149688:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877953.149695:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511438592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877953.149701:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877953.149704:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877953.149706:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877953.149709:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2939 00000100:00080000:1.0:1788877953.149711:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877953.149716:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788877953.149718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788877953.149724:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877953.149737:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511438720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877953.149767:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877953/real 1788877953] req@ffffa0618869a680 x1875771511438208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877953.149790:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511438208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877953.149799:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877953.149804:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877953.149809:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511438208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877953.149838:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877953.149867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877953.149878:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877953/real 1788877953] req@ffffa0618869bb80 x1875771511438336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877953.149887:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511438336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877953.149894:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877953.149896:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877953.149899:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877953.149904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877953.149922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877953.149935:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877953/real 1788877953] req@ffffa0618869ad80 x1875771511438464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877953.149944:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511438464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877953.149954:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877953.149957:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877953.149959:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877953.149963:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877953.149974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877953.149980:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877953/real 1788877953] req@ffffa06188698000 x1875771511438592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877953.149987:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511438592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877953.149993:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877953.149995:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877953.149997:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877953.150001:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877953.150011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877953.150017:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877953/real 1788877953] req@ffffa06188699f80 x1875771511438720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877953.150024:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511438720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877953.150029:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877953.150031:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877953.150033:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877953.789491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877953.789549:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877954.045463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877954.813496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877954.813595:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877955.069499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877955.837479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877955.837577:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877956.093530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877956.861469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877956.861560:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877957.117488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877957.885491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877957.885530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877958.141448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877958.269489:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877958.269500:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877958.269504:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877958.269515:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2944 00000100:00080000:2.0:1788877958.269526:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877958.269535:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877958.269570:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511438848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877958.269593:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877958.269596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877958.269598:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877958.269601:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2944 00000100:00080000:2.0:1788877958.269604:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877958.269608:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877958.269617:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788877958.269627:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511438848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877958.269628:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511438976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877958.269642:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877958.269644:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877958.269645:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877958.269648:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2944 00000100:00080000:2.0:1788877958.269650:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877958.269653:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877958.269656:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877958.269666:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511439104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877958.269672:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877958.269675:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877958.269676:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877958.269678:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2944 00000100:00080000:2.0:1788877958.269680:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877958.269683:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877958.269686:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788877958.269690:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877958.269698:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511439232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877958.269705:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877958.269708:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877958.269709:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877958.269712:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2944 00000100:00080000:2.0:1788877958.269715:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877958.269717:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877958.269721:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877958.269724:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877958/real 1788877958] req@ffffa061888ad500 x1875771511438848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877958.269730:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511439360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877958.269748:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511438848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877958.269758:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877958.269761:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877958.269764:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511438848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877958.269781:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877958.269795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877958.269802:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877958/real 1788877958] req@ffffa061888adf80 x1875771511438976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877958.269810:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511438976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877958.269819:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877958.269823:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877958.269827:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511438976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877958.269833:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877958.269852:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877958.269863:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877958/real 1788877958] req@ffffa061888ac000 x1875771511439104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877958.269876:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511439104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877958.269882:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877958.269884:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877958.269886:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877958.269890:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877958.269905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877958.269912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877958/real 1788877958] req@ffffa061888af480 x1875771511439232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877958.269920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511439232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877958.269925:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877958.269927:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877958.269929:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877958.269933:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877958.269943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877958.269948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877958/real 1788877958] req@ffffa061888ae300 x1875771511439360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877958.269955:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511439360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877958.269960:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877958.269962:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877958.269964:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877958.909447:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877958.909551:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877959.165459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877959.933451:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877959.933492:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877960.189447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877960.957445:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877960.957496:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877961.213459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877961.981441:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877961.981503:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877962.237460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877963.005442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877963.005519:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877963.261468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877963.389521:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877963.389533:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877963.389538:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877963.389549:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2950 00000100:00080000:3.0:1788877963.389561:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877963.389571:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877963.389610:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511439488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877963.389640:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877963.389644:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877963.389646:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877963.389651:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2950 00000100:00080000:3.0:1788877963.389655:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877963.389660:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877963.389671:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788877963.389685:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511439488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877963.389692:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511439616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877963.389715:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877963.389718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877963.389720:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877963.389724:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2950 00000100:00080000:3.0:1788877963.389728:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877963.389732:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877963.389737:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877963.389749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511439744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877963.389759:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877963.389765:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788877963.389767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877963.389769:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877963.389778:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2950 00000100:00080000:3.0:1788877963.389783:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877963.389788:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877963.389794:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877963.389809:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511439872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877963.389818:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877963/real 1788877963] req@ffffa06188871500 x1875771511439488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877963.389825:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877963.389838:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877963.389843:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877963.389854:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2950 00000100:00080000:3.0:1788877963.389861:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877963.389872:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877963.389883:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877963.389921:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511440000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877963.389923:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511439488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877963.389968:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877963.389993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877963.389998:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511439488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877963.390028:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877963.390084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877963.390099:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877963/real 1788877963] req@ffffa06188873100 x1875771511439616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877963.390126:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511439616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877963.390135:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877963.390139:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877963.390142:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877963.390148:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877963.390168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877963.390179:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877963/real 1788877963] req@ffffa06188872a00 x1875771511439744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877963.390190:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511439744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877963.390229:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877963.390233:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877963.390237:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877963.390243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877963.390267:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877963.390279:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877963/real 1788877963] req@ffffa06188873800 x1875771511439872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877963.390291:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511439872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877963.390299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877963.390302:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877963.390306:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511439872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877963.390311:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877963.390326:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877963.390335:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877963/real 1788877963] req@ffffa06188873480 x1875771511440000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877963.390346:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511440000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877963.390354:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877963.390357:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877963.390361:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877964.029463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877964.029512:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877964.285476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877965.053485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877965.053550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877965.309481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877966.077494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877966.077560:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877966.333478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877967.101557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877967.101599:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877967.357580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877968.125488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877968.125514:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877968.381451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877968.509462:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877968.509485:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877968.509491:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877968.509502:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2955 00000100:00080000:3.0:1788877968.509515:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877968.509524:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877968.509574:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511440128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877968.509603:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877968.509608:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877968.509610:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877968.509616:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2955 00000100:00080000:3.0:1788877968.509621:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877968.509627:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788877968.509635:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511440128:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788877968.509638:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877968.509657:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511440256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877968.509669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877968.509676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877968.509678:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877968.509682:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2955 00000100:00080000:3.0:1788877968.509687:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788877968.509705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877968.509706:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877968.509715:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877968.509730:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511440384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877968.509738:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877968/real 1788877968] req@ffffa06188872300 x1875771511440128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877968.509748:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877968.509753:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877968.509756:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788877968.509757:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511440128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877968.509768:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2955 00000100:00080000:0.0:1788877968.509771:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788877968.509775:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788877968.509777:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877968.509783:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511440128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788877968.509786:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877968.509795:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877968.509810:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511440512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877968.509813:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440256:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788877968.509831:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877968.509835:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788877968.509839:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877968.509840:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877968.509848:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2955 00000100:00100000:0.0:1788877968.509854:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877968/real 1788877968] req@ffffa06188873b80 x1875771511440256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877968.509854:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877968.509866:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788877968.509871:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511440256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877968.509890:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877968.509892:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788877968.509894:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877968.509896:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877968.509902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788877968.509921:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511440640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788877968.509924:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877968.509934:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877968/real 1788877968] req@ffffa06188870700 x1875771511440384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877968.509946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511440384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877968.509952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877968.509956:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877968.509958:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877968.509968:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877968.509990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877968.509998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877968/real 1788877968] req@ffffa06188872680 x1875771511440512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877968.510005:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511440512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877968.510010:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877968.510012:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877968.510015:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877968.510018:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877968.510040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877968.510046:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877968/real 1788877968] req@ffffa06188872d80 x1875771511440640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877968.510053:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511440640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877968.510057:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877968.510059:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877968.510061:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877969.149458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877969.149516:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877969.405491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877970.173453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877970.173518:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877970.429476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877971.197446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877971.197525:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877971.453490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877972.221442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877972.221527:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877972.477441:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877973.245481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877973.245545:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877973.501452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877973.629490:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877973.629498:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877973.629501:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877973.629510:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2960 00000100:00080000:2.0:1788877973.629520:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877973.629527:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877973.629560:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511440768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877973.629588:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877973.629592:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877973.629594:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877973.629599:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2960 00000100:00080000:2.0:1788877973.629604:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877973.629610:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788877973.629616:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511440768:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788877973.629622:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877973.629636:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511440896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877973.629644:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877973.629647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877973.629649:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877973.629652:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2960 00000100:00080000:2.0:1788877973.629655:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877973.629658:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877973.629662:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877973.629671:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511441024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877973.629677:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877973.629680:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877973.629681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877973.629684:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2960 00000100:00080000:2.0:1788877973.629687:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877973.629690:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877973.629693:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877973.629699:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877973.629703:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511441152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877973.629713:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877973.629716:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877973.629718:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877973.629720:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2960 00000100:00080000:2.0:1788877973.629723:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877973.629726:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877973.629729:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877973.629735:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877973.629741:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511441280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877973.629773:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877973.629798:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877974.269453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877974.269511:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877974.525456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877975.293453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877975.293518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877975.549461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877976.317462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877976.317522:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877976.573466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788877976.701600:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788877976.701636:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788877976.701650:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877976.701681:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877976.701691:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877976.701702:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788877976.701713:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788877976.701722:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877973/real 1788877976] req@ffffa061888afb80 x1875771511440896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877976.701749:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511440896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877976.701758:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877976.701762:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877976.701765:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511440896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877976.701771:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877973/real 1788877976] req@ffffa061888af100 x1875771511441024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877976.701782:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511441024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877976.701790:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877976.701793:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877976.701797:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877976.701804:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877973/real 1788877976] req@ffffa061888ae680 x1875771511440768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877976.701823:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511440768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877976.701837:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877976.701841:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877976.701844:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511440768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877976.701869:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877973/real 1788877976] req@ffffa061888adc00 x1875771511441152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877976.701882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511441152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877976.701890:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877976.701894:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877976.701896:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877976.701902:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877973/real 1788877976] req@ffffa061888aca80 x1875771511441280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877976.701913:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511441280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788877998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877976.701920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877976.701924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877976.701927:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877977.341480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877977.341608:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877977.597486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877978.365463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877978.365533:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877978.621467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877978.749513:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877978.749523:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877978.749528:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877978.749538:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2965 00000100:00080000:1.0:1788877978.749549:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877978.749556:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877978.749594:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511441408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877978.749617:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877978.749621:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877978.749622:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877978.749626:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2965 00000100:00080000:1.0:1788877978.749629:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877978.749631:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877978.749641:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877978.749651:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511441536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877978.749659:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877978.749662:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877978.749663:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877978.749666:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2965 00000100:00080000:1.0:1788877978.749669:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788877978.749673:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511441408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877978.749674:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877978.749679:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877978.749687:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511441664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877978.749694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877978.749697:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877978.749698:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877978.749700:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2965 00000100:00080000:1.0:1788877978.749702:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877978.749704:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877978.749708:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877978.749715:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511441792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877978.749726:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877978.749729:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877978.749732:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877978.749737:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2965 00000100:00080000:1.0:1788877978.749753:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877978.749758:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877978.749764:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788877978.749772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877978.749779:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511441920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877978.749818:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877978/real 1788877978] req@ffffa06188699f80 x1875771511441408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877978.749843:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511441408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877978.749856:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877978.749861:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877978.749865:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511441408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877978.749896:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877978.749921:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877978.749934:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877978/real 1788877978] req@ffffa06188698000 x1875771511441536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877978.749949:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511441536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877978.749959:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877978.749963:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877978.749967:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877978.749974:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877978.750016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877978.750029:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877978/real 1788877978] req@ffffa0618869ad80 x1875771511441664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877978.750043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511441664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877978.750052:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877978.750056:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877978.750060:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877978.750067:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877978.750089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877978.750101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877978/real 1788877978] req@ffffa0618869bb80 x1875771511441792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877978.750113:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511441792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877978.750122:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877978.750127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877978.750148:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877978.750156:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877978.750178:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877978.750190:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877978/real 1788877978] req@ffffa0618869a680 x1875771511441920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877978.750212:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511441920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877978.750221:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877978.750224:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877978.750228:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511441920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877979.389519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877979.389579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877979.645531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877980.413457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877980.413515:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877980.669475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877981.437489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877981.437556:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877981.693467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877982.461456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877982.461593:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877982.717448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877983.485461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877983.485512:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877983.741483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877983.869514:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877983.869523:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877983.869527:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877983.869536:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2970 00000100:00080000:1.0:1788877983.869545:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877983.869552:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877983.869591:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511442048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877983.869618:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877983.869622:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877983.869624:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877983.869628:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2970 00000100:00080000:1.0:1788877983.869632:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877983.869636:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877983.869645:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788877983.869646:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511442048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788877983.869659:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511442176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877983.869668:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877983.869678:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877983.869679:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877983.869683:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2970 00000100:00080000:1.0:1788877983.869694:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877983.869697:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877983.869701:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788877983.869704:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877983.869713:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511442304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877983.869721:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877983.869724:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877983.869725:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877983.869728:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2970 00000100:00080000:1.0:1788877983.869731:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877983.869735:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788877983.869738:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877983/real 1788877983] req@ffffa0618869b800 x1875771511442048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788877983.869740:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877983.869753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511442432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877983.869760:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511442048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788877983.869762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877983.869770:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877983.869773:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788877983.869780:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788877983.869780:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2970 00000100:00080000:1.0:1788877983.869786:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788877983.869788:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788877983.869792:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788877983.869796:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511442048:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788877983.869797:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877983.869809:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511442560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877983.869821:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877983.869838:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877983.869846:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877983/real 1788877983] req@ffffa0618869aa00 x1875771511442176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877983.869855:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511442176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877983.869861:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877983.869863:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877983.869866:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877983.869869:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877983.869880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877983.869886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877983/real 1788877983] req@ffffa0618869b100 x1875771511442304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877983.869893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511442304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877983.869899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877983.869900:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877983.869902:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877983.869906:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877983.869930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877983.869938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877983/real 1788877983] req@ffffa06188698380 x1875771511442432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877983.869946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511442432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877983.869951:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877983.869953:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877983.869955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877983.869959:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877983.869968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877983.869974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877983/real 1788877983] req@ffffa06188699c00 x1875771511442560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877983.869981:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511442560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877983.869986:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877983.869988:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877983.869990:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877984.509468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877984.509532:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877984.765449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877985.533492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877985.533734:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877985.789473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877986.557488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877986.557568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877986.813449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877987.581492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877987.581570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877987.837476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877988.605538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877988.605591:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877988.861462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788877988.989469:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877988.989477:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788877988.989481:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877988.989490:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2975 00000100:00080000:1.0:1788877988.989517:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877988.989525:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788877988.989562:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511442688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877988.989589:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877988.989592:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788877988.989594:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877988.989598:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2975 00000100:00080000:1.0:1788877988.989600:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877988.989615:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877988.989623:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877988.989640:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511442816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877988.989648:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877988.989667:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788877988.989669:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877988.989672:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2975 00000100:00080000:1.0:1788877988.989675:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788877988.989680:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511442688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788877988.989681:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877988.989689:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788877988.989698:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511442944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877988.989706:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877988.989709:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788877988.989710:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877988.989712:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2975 00000100:00080000:1.0:1788877988.989715:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877988.989717:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877988.989720:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788877988.989729:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511443072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788877988.989735:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788877988.989737:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788877988.989739:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788877988.989742:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2975 00000100:00080000:1.0:1788877988.989744:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788877988.989746:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788877988.989749:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788877988.989754:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788877988.989761:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511443200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788877988.989807:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877988/real 1788877988] req@ffffa06188699c00 x1875771511442688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877988.989841:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511442688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877988.989853:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877988.989857:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877988.989862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511442688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877988.989871:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877988.989896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877988.989910:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877988/real 1788877988] req@ffffa06188698380 x1875771511442816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877988.989925:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511442816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877988.989940:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877988.989944:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877988.989948:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877988.989973:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788877988.989997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877988.990011:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877988/real 1788877988] req@ffffa0618869b100 x1875771511442944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877988.990026:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511442944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877988.990035:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877988.990039:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877988.990042:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511442944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877988.990050:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877988.990074:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877988.990108:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877988/real 1788877988] req@ffffa0618869aa00 x1875771511443072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877988.990122:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511443072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877988.990131:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877988.990135:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877988.990139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788877988.990145:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788877988.990167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788877988.990179:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877988/real 1788877988] req@ffffa0618869b800 x1875771511443200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788877988.990192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511443200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877988.990200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877988.990204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788877988.990207:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877989.629530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877989.629633:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877989.885554:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877990.653546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877990.653681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877990.909562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877991.677504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877991.677640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877991.933489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877992.701491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877992.701580:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877992.957491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877993.725502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877993.725626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877993.981528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788877994.109564:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877994.109581:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788877994.109590:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877994.109608:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2980 00000100:00080000:2.0:1788877994.109626:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877994.109639:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788877994.109696:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511443328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877994.109744:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877994.109752:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788877994.109756:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877994.109765:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2980 00000100:00080000:2.0:1788877994.109774:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877994.109785:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877994.109801:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788877994.109806:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511443328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788877994.109835:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511443456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788877994.109860:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788877994.109867:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788877994.109871:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877994.109879:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2980 00000100:00080000:2.0:1788877994.109887:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788877994.109895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877994.109903:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877994.109917:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788877994.109944:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511443584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877994.109952:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877994/real 1788877994] req@ffffa061888ac380 x1875771511443328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877994.109996:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788877994.110004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511443328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877994.110008:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788877994.110017:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788877994.110035:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788877994.110037:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2980 00000100:00080000:0.0:1788877994.110049:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788877994.110053:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788877994.110056:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511443328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788877994.110100:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788877994.110113:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788877994.110120:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877994.110148:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511443712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788877994.110167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788877994.110188:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788877994.110197:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877994/real 1788877994] req@ffffa061888aea00 x1875771511443456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877994.110201:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788877994.110210:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788877994.110231:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2980 00000100:00100000:0.0:1788877994.110236:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511443456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788877994.110245:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788877994.110264:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788877994.110270:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788877994.110278:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788877994.110280:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877994.110290:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877994.110302:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788877994.110316:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511443840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788877994.110340:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877994.110433:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877994/real 1788877994] req@ffffa061888ad880 x1875771511443584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877994.110452:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511443584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877994.110463:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877994.110468:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877994.110472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877994.110493:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877994.110523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877994.110537:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877994/real 1788877994] req@ffffa061888aed80 x1875771511443712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877994.110551:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511443712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877994.110561:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877994.110565:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877994.110570:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877994.110578:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877994.110596:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877994.110607:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877994/real 1788877994] req@ffffa061888ac700 x1875771511443840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877994.110621:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511443840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877994.110635:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877994.110640:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877994.110648:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511443840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877994.749508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877994.749603:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877995.005504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877995.773542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877995.773720:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877996.029520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877996.797524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877996.797597:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877997.053555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877997.821483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877997.821697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877998.077479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788877998.845474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788877998.845574:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788877999.101488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788877999.229508:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877999.229519:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788877999.229523:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877999.229534:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2985 00000100:00080000:3.0:1788877999.229545:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877999.229552:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788877999.229591:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511443968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877999.229619:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877999.229622:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788877999.229624:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877999.229628:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2985 00000100:00080000:3.0:1788877999.229632:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877999.229636:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877999.229647:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788877999.229658:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511443968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788877999.229664:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511444096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877999.229677:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877999.229680:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788877999.229682:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877999.229685:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2985 00000100:00080000:3.0:1788877999.229688:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877999.229692:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877999.229697:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788877999.229708:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511444224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877999.229716:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877999.229719:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788877999.229721:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788877999.229728:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2985 00000800:00000100:0.0:1788877999.229730:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788877999.229733:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877999.229738:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877999.229744:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877999.229755:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511444352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788877999.229763:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788877999.229766:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788877999.229769:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788877999.229774:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877999/real 1788877999] req@ffffa06188871c00 x1875771511443968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788877999.229777:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2985 00000100:00080000:3.0:1788877999.229785:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788877999.229793:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788877999.229801:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788877999.229817:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511444480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788877999.229825:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511443968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877999.229869:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877999.229888:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877999.229896:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511443968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788877999.229925:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877999.229960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877999.229974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877999/real 1788877999] req@ffffa06188870380 x1875771511444096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877999.229986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511444096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877999.229995:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877999.229998:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877999.230001:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877999.230007:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788877999.230024:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877999.230033:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877999/real 1788877999] req@ffffa06188870a80 x1875771511444224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877999.230069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511444224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877999.230090:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877999.230095:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877999.230101:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788877999.230110:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877999.230135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877999.230146:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877999/real 1788877999] req@ffffa06188871880 x1875771511444352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877999.230156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511444352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877999.230164:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877999.230167:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877999.230170:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877999.230175:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788877999.230188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788877999.230196:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788877999/real 1788877999] req@ffffa06188872d80 x1875771511444480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788877999.230205:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511444480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788877999.230212:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788877999.230214:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788877999.230218:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788877999.869528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788877999.869643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878000.125489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878000.893474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878000.893561:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878001.149491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878001.917555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878001.917672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878002.173484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878002.941484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878002.941688:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878003.197489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878003.965514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878003.965669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878004.221446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878004.349511:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878004.349524:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878004.349531:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878004.349546:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2991 00000100:00080000:3.0:1788878004.349561:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878004.349572:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878004.349666:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511444608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878004.349714:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878004.349720:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878004.349723:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878004.349730:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2991 00000100:00080000:3.0:1788878004.349737:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878004.349759:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878004.349774:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878004.349797:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511444608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878004.349798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511444736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878004.349816:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878004.349821:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878004.349824:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878004.349829:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2991 00000100:00080000:3.0:1788878004.349834:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878004.349841:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878004.349849:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878004.349879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878004.349879:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511444864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878004.349895:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878004.349900:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878004.349903:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878004.349908:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2991 00000100:00080000:3.0:1788878004.349914:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878004.349920:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878004.349924:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878004/real 1788878004] req@ffffa06188872680 x1875771511444608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878004.349928:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878004.349960:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511444992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878004.349969:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511444608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878004.349973:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878004.349978:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878004.349981:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878004.349984:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878004.349985:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2991 00000100:00080000:0.0:1788878004.349989:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878004.349990:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878004.349995:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878004.349996:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511444608:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878004.350002:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878004.350043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878004.350044:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511445120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878004.350078:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878004.350098:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878004/real 1788878004] req@ffffa06188870700 x1875771511444736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878004.350127:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511444736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878004.350137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878004.350140:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878004.350143:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878004.350150:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878004.350173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878004.350184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878004/real 1788878004] req@ffffa06188873b80 x1875771511444864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878004.350194:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511444864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878004.350202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878004.350204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878004.350207:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878004.350213:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878004.350229:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878004.350237:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878004/real 1788878004] req@ffffa06188872300 x1875771511444992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878004.350248:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511444992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878004.350255:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878004.350258:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878004.350261:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511444992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878004.350278:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878004.350292:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878004.350301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878004/real 1788878004] req@ffffa06188873480 x1875771511445120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878004.350311:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511445120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878004.350318:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878004.350321:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878004.350324:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878004.989482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878004.989589:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878005.245521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878006.013509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878006.013616:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878006.269458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878007.037452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878007.037529:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878007.293493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878008.061502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878008.061664:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878008.317500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878009.085517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878009.085663:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878009.341521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878009.469529:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878009.469543:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878009.469562:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878009.469578:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 2996 00000100:00080000:3.0:1788878009.469594:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878009.469615:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878009.469667:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511445248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878009.469697:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878009.469702:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878009.469705:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878009.469711:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2996 00000100:00080000:3.0:1788878009.469717:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878009.469722:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878009.469735:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878009.469755:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511445248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878009.469756:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511445376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878009.469786:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878009.469790:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878009.469793:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878009.469797:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2996 00000100:00080000:3.0:1788878009.469801:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878009.469817:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878009.469823:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878009.469837:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511445504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878009.469859:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878009.469863:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878009.469866:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878009.469871:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2996 00000100:00080000:3.0:1788878009.469875:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878009.469880:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878009.469887:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878009.469900:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511445632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878009.469912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878009.469930:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878009.469939:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878009.469943:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878009.469952:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 2996 00000100:00080000:3.0:1788878009.469960:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878009.469969:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878009.469980:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878009.470029:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878009/real 1788878009] req@ffffa06188873800 x1875771511445248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878009.470045:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511445760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878009.470127:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511445248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878009.470155:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878009.470165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878009.470174:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511445248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878009.470234:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878009.470287:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878009.470317:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878009/real 1788878009] req@ffffa06188872a00 x1875771511445376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878009.470345:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511445376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878009.470365:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878009.470371:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878009.470379:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878009.470434:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878009.470482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878009.470509:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878009/real 1788878009] req@ffffa06188873100 x1875771511445504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878009.470542:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511445504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878009.470572:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878009.470580:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878009.470587:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878009.470599:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878009.470650:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878009.470672:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878009/real 1788878009] req@ffffa06188871500 x1875771511445632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878009.470696:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511445632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878009.470713:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878009.470720:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878009.470727:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878009.470739:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878009.470779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878009.470801:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878009/real 1788878009] req@ffffa06188873480 x1875771511445760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878009.470826:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511445760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878009.470843:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878009.470850:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878009.470859:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511445760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878010.109507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878010.109654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878010.365514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878011.133568:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878011.133654:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878011.389470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878012.157473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878012.157570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878012.413473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878013.181470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878013.181625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878013.437522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878014.205514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878014.205550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878014.461534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878014.589509:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878014.589520:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878014.589524:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878014.589535:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3001 00000100:00080000:2.0:1788878014.589546:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878014.589554:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878014.589591:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511445888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878014.589623:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878014.589627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878014.589630:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878014.589633:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3001 00000100:00080000:2.0:1788878014.589637:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878014.589642:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878014.589669:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878014.589687:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511445888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878014.589695:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511446016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878014.589712:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878014.589716:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878014.589718:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878014.589723:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3001 00000100:00080000:2.0:1788878014.589727:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878014.589732:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878014.589738:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878014.589753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511446144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878014.589765:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878014.589769:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878014.589771:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878014.589779:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3001 00000800:00000100:0.0:1788878014.589783:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878014.589787:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878014.589793:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878014.589800:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878014.589846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511446272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878014.589854:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878014/real 1788878014] req@ffffa061888ad180 x1875771511445888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878014.589866:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878014.589878:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878014.589883:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878014.589905:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3001 00000100:00080000:2.0:1788878014.589909:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878014.589912:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511445888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878014.589919:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878014.589929:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788878014.589931:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878014.589936:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878014.589942:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511445888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878014.589985:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511446400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878014.589998:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878014.590023:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878014.590034:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878014/real 1788878014] req@ffffa061888ace00 x1875771511446016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878014.590043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511446016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878014.590051:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878014.590053:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878014.590056:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878014.590060:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878014.590073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878014.590079:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878014/real 1788878014] req@ffffa061888ac700 x1875771511446144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878014.590088:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511446144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878014.590093:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878014.590095:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878014.590097:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878014.590101:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878014.590113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878014.590119:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878014/real 1788878014] req@ffffa061888aed80 x1875771511446272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878014.590128:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511446272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878014.590133:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878014.590135:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878014.590138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878014.590149:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878014.590160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878014.590167:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878014/real 1788878014] req@ffffa061888ad880 x1875771511446400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878014.590175:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511446400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878014.590180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878014.590182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878014.590184:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878015.229482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878015.229661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878015.485500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878016.253498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878016.253602:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878016.509509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878017.277461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878017.277538:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878017.533559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788878018.301581:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878018.301582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788878018.557574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878019.325469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878019.325564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878019.581468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878019.709452:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878019.709465:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878019.709471:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878019.709484:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3006 00000100:00080000:3.0:1788878019.709496:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878019.709505:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878019.709559:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511446528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878019.709582:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878019.709585:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878019.709586:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878019.709590:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3006 00000100:00080000:3.0:1788878019.709593:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878019.709597:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878019.709606:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878019.709613:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511446528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878019.709618:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511446656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878019.709628:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878019.709630:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878019.709632:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878019.709635:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3006 00000100:00080000:3.0:1788878019.709638:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878019.709640:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878019.709644:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878019.709653:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511446784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878019.709660:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878019.709663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878019.709665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878019.709667:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3006 00000100:00080000:3.0:1788878019.709670:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878019.709676:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878019.709677:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788878019.709683:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878019.709692:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511446912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878019.709699:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878019.709702:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878019.709704:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878019.709707:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3006 00000100:00080000:3.0:1788878019.709710:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878019.709714:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878019/real 1788878019] req@ffffa06188872300 x1875771511446528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878019.709715:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878019.709722:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878019.709734:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511447040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878019.709746:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511446528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878019.709754:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878019.709758:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878019.709761:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511446528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878019.709779:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878019.709795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878019.709803:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878019/real 1788878019] req@ffffa06188873b80 x1875771511446656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878019.709811:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511446656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878019.709817:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878019.709820:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878019.709822:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878019.709842:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878019.709855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878019.709862:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878019/real 1788878019] req@ffffa06188870700 x1875771511446784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878019.709870:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511446784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878019.709880:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878019.709883:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878019.709885:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878019.709889:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878019.709901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878019.709906:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878019/real 1788878019] req@ffffa06188872680 x1875771511446912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878019.709914:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511446912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878019.709920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878019.709921:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878019.709923:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511446912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878019.709927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878019.709938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878019.709961:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878019/real 1788878019] req@ffffa06188872d80 x1875771511447040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878019.709991:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511447040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878019.710000:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878019.710003:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878019.710007:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878020.349564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878020.349733:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878020.605508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878021.373545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878021.373720:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878021.629541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878022.397497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878022.397556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878022.653508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878023.421515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878023.421644:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878023.677492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878024.445514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878024.445642:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878024.701474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878024.829486:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878024.829498:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878024.829504:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878024.829517:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3011 00000100:00080000:3.0:1788878024.829531:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878024.829542:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878024.829588:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511447168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878024.829632:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878024.829639:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878024.829642:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878024.829650:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3011 00000100:00080000:3.0:1788878024.829658:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878024.829668:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878024.829682:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511447168:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878024.829684:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878024.829714:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511447296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878024.829729:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878024.829733:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878024.829736:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878024.829740:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3011 00000100:00080000:3.0:1788878024.829748:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878024.829753:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878024.829759:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878024.829794:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511447424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878024.829800:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878024.829816:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878024.829822:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878024.829825:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878024.829831:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3011 00000100:00080000:3.0:1788878024.829837:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878024.829844:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878024.829850:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878024.829864:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878024/real 1788878024] req@ffffa06188871880 x1875771511447168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878024.829865:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511447552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878024.829908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511447168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878024.829908:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878024.829919:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878024.829939:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878024.829948:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3011 00000100:00080000:0.0:1788878024.829952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878024.829954:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878024.829962:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878024.829966:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788878024.829969:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878024.829977:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511447168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878024.829987:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511447680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878024.830012:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878024.830088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878024.830106:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878024/real 1788878024] req@ffffa06188870a80 x1875771511447296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878024.830122:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511447296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878024.830134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878024.830139:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878024.830144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878024.830151:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878024.830172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878024.830182:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878024/real 1788878024] req@ffffa06188870380 x1875771511447424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878024.830192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511447424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878024.830199:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878024.830201:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878024.830204:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878024.830208:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878024.830222:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878024.830229:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878024/real 1788878024] req@ffffa06188871c00 x1875771511447552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878024.830237:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511447552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878024.830243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878024.830245:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878024.830266:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878024.830272:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878024.830290:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878024.830299:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878024/real 1788878024] req@ffffa06188872d80 x1875771511447680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878024.830311:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511447680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878024.830324:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878024.830330:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878024.830336:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878025.469534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878025.469669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878025.725464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878026.493493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878026.493597:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878026.749457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878027.517485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878027.517579:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878027.773481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878028.541505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878028.541558:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878028.797491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878029.565498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878029.565760:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878029.821516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878029.949580:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878029.949592:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878029.949598:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878029.949611:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3016 00000100:00080000:1.0:1788878029.949626:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878029.949637:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878029.949685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511447808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878029.949725:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878029.949729:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878029.949732:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878029.949737:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3016 00000100:00080000:1.0:1788878029.949754:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878029.949760:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878029.949774:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878029.949785:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511447808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878029.949796:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511447936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878029.949814:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878029.949818:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878029.949821:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878029.949826:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3016 00000100:00080000:1.0:1788878029.949831:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878029.949837:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878029.949844:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878029.949859:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511448064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878029.949883:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878029.949887:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878029.949890:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878029.949927:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3016 00000800:00000100:0.0:1788878029.949929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878029.949938:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878029.949947:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878029.949955:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878029.949972:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511448192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878029.949987:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878029/real 1788878029] req@ffffa0618869a680 x1875771511447808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878029.950000:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878029.950013:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878029.950019:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878029.950031:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3016 00000100:00080000:1.0:1788878029.950041:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878029.950052:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878029.950060:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878029.950067:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511447808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878029.950081:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511448320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878029.950088:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878029.950103:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878029.950113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511447808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878029.950144:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878029.950191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878029.950208:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878029/real 1788878029] req@ffffa0618869bb80 x1875771511447936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878029.950224:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511447936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878029.950235:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878029.950239:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878029.950244:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511447936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878029.950251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878029.950273:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878029.950285:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878029/real 1788878029] req@ffffa0618869ad80 x1875771511448064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878029.950299:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511448064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878029.950317:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878029.950321:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878029.950325:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878029.950332:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878029.950351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878029.950362:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878029/real 1788878029] req@ffffa06188698000 x1875771511448192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878029.950375:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511448192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878029.950417:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878029.950421:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878029.950425:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878029.950433:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878029.950456:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878029.950469:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878029/real 1788878029] req@ffffa06188699f80 x1875771511448320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878029.950484:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511448320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878029.950493:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878029.950497:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878029.950502:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878030.589497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878030.589619:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878030.845482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878031.613560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878031.613723:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878031.869542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878032.637499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878032.637593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878032.893508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878033.661543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878033.661659:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878033.917513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878034.685494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878034.685665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878034.941597:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878035.069562:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878035.069575:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878035.069581:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878035.069596:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3021 00000100:00080000:1.0:1788878035.069611:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878035.069621:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878035.069672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511448448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878035.069707:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878035.069712:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878035.069714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878035.069720:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3021 00000100:00080000:1.0:1788878035.069725:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878035.069740:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878035.069753:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878035.069770:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511448448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878035.069772:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511448576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878035.069820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878035.069824:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878035.069826:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878035.069832:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3021 00000100:00080000:1.0:1788878035.069838:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878035.069843:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878035.069851:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878035.069869:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511448704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878035.069912:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878035.069934:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878035.069936:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878035.069941:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3021 00000100:00080000:1.0:1788878035.069946:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878035.069954:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878035.069957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788878035.069963:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878035.069982:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511448832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878035.069995:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878035.069999:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878035.070001:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878035.070007:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3021 00000100:00080000:1.0:1788878035.070013:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878035.070014:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878035/real 1788878035] req@ffffa06188699f80 x1875771511448448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878035.070022:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878035.070035:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878035.070054:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511448960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878035.070060:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511448448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878035.070091:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878035.070097:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878035.070103:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511448448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878035.070133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878035.070162:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878035.070177:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878035/real 1788878035] req@ffffa06188698000 x1875771511448576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878035.070191:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511448576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878035.070202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878035.070206:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878035.070210:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878035.070216:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878035.070234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878035.070244:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878035/real 1788878035] req@ffffa0618869ad80 x1875771511448704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878035.070257:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511448704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878035.070267:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878035.070270:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878035.070273:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878035.070280:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878035.070297:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878035.070307:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878035/real 1788878035] req@ffffa0618869bb80 x1875771511448832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878035.070320:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511448832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878035.070328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878035.070331:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878035.070336:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878035.070341:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878035.070358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878035.070368:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878035/real 1788878035] req@ffffa0618869a680 x1875771511448960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878035.070437:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511448960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878035.070449:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878035.070453:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878035.070457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511448960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878035.709508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878035.709611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878035.965575:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878036.733530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878036.733601:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878036.989481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878037.757525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878037.757654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878038.013513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878038.781446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878038.781537:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878039.037444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878039.805503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878039.805628:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878040.061519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878040.189521:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878040.189534:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878040.189541:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878040.189556:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3026 00000100:00080000:1.0:1788878040.189570:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878040.189581:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878040.189627:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511449088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878040.189665:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878040.189670:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878040.189672:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878040.189677:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3026 00000100:00080000:1.0:1788878040.189683:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878040.189689:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878040.189703:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878040.189723:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511449216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878040.189736:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511449088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878040.189739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878040.189755:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878040.189759:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878040.189763:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3026 00000100:00080000:1.0:1788878040.189767:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878040.189772:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878040.189779:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878040.189796:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511449344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878040.189809:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878040.189813:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878040.189816:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878040.189820:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3026 00000100:00080000:1.0:1788878040.189827:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878040.189831:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878040.189837:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878040.189850:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511449472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878040.189890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878040.189895:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878040.189899:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878040.189907:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3026 00000100:00080000:1.0:1788878040.189916:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878040.189927:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878040.189934:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449216:192.168.203.111@tcp:38:kworker.0 00000008:00100000:1.0:1788878040.189943:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878040.189983:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511449600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878040.190020:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878040.190057:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878040.190135:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878040.829561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878040.829605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878041.085500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878041.853513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878041.853623:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878042.109486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878042.877588:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878042.877769:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878043.133489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788878043.261754:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788878043.261863:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788878043.261907:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878043.261972:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878043.261990:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878043.262006:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878043.262019:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878043.262110:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878040/real 1788878043] req@ffffa06188699c00 x1875771511449600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878043.262162:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511449600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878043.262185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878043.262194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878043.262204:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878043.262219:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878040/real 1788878043] req@ffffa0618869aa00 x1875771511449216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878043.262237:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511449216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878043.262248:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878043.262253:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878043.262258:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878043.262266:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878040/real 1788878043] req@ffffa0618869b100 x1875771511449344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878043.262289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511449344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878043.262299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878043.262306:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878043.262312:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878043.262324:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878040/real 1788878043] req@ffffa06188698380 x1875771511449472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878043.262348:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511449472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878043.262362:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878043.262369:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878043.262374:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878043.262434:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878040/real 1788878043] req@ffffa0618869b800 x1875771511449088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878043.262468:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511449088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878043.262488:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878043.262496:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878043.262504:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511449088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878043.901542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878043.901647:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878044.157517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878044.925516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878044.925689:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878045.181461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878045.309605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878045.309618:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878045.309623:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878045.309649:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3032 00000100:00080000:3.0:1788878045.309662:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878045.309671:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878045.309745:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511449728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878045.309806:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878045.309814:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878045.309818:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878045.309827:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3032 00000100:00080000:3.0:1788878045.309898:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878045.309901:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511449728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878045.309912:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878045.309952:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878045.309983:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511449856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878045.310004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878045.310006:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878045.310014:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878045.310019:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878045.310027:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3032 00000100:00080000:3.0:1788878045.310035:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878045.310043:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878045.310052:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878045.310062:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878045/real 1788878045] req@ffffa06188872680 x1875771511449728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878045.310076:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511449984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878045.310113:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511449728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878045.310131:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878045.310142:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878045.310144:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878045.310155:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788878045.310158:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511449728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878045.310161:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878045.310173:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3032 00000100:00080000:3.0:1788878045.310184:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878045.310191:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878045.310201:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878045.310206:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878045.310226:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511450112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878045.310243:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878045.310252:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788878045.310254:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878045.310257:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878045.310268:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3032 00000100:00080000:3.0:1788878045.310275:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878045.310286:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878045.310288:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878045/real 1788878045] req@ffffa06188870700 x1875771511449856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878045.310296:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878045.310318:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511450240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878045.310332:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511449856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878045.310352:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878045.310359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878045.310366:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878045.310378:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878045.310457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878045.310478:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878045/real 1788878045] req@ffffa06188873b80 x1875771511449984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878045.310501:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511449984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878045.310514:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878045.310519:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878045.310525:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511449984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878045.310550:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878045.310585:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878045.310602:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878045/real 1788878045] req@ffffa06188872300 x1875771511450112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878045.310620:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511450112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878045.310632:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878045.310636:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878045.310650:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878045.310660:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878045.310687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878045.310705:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878045/real 1788878045] req@ffffa06188873480 x1875771511450240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878045.310756:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511450240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878045.310770:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878045.310775:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878045.310780:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878045.949526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878045.949740:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878046.205449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878046.973566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878046.973690:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878047.229531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878047.997543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878047.997632:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878048.253524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878049.021564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878049.021680:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878049.277527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878050.045527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878050.045609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878050.301519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878050.429574:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878050.429588:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878050.429594:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878050.429608:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3037 00000100:00080000:2.0:1788878050.429622:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878050.429633:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878050.429682:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511450368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878050.429722:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878050.429727:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878050.429730:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878050.429736:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3037 00000100:00080000:2.0:1788878050.429743:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878050.429751:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878050.429768:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878050.429787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511450368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878050.429800:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511450496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878050.429850:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878050.429857:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878050.429861:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878050.429870:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3037 00000100:00080000:2.0:1788878050.429878:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878050.429887:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878050.429896:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878050.429918:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511450624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878050.429932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878050.429941:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878050.429947:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878050.429951:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878050.429958:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3037 00000100:00080000:2.0:1788878050.429965:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878050.429971:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878050.429979:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878050.429994:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511450752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878050.430000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878050/real 1788878050] req@ffffa061888aea00 x1875771511450368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878050.430029:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878050.430039:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878050.430042:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878050.430049:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3037 00000100:00080000:2.0:1788878050.430056:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878050.430058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511450368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878050.430066:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878050.430080:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878050.430092:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511450880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878050.430116:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878050.430128:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878050.430135:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511450368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878050.430171:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878050.430205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878050.430223:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878050/real 1788878050] req@ffffa061888ac380 x1875771511450496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878050.430240:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511450496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878050.430251:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878050.430256:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878050.430260:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878050.430267:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878050.430288:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878050.430300:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878050/real 1788878050] req@ffffa061888aca80 x1875771511450624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878050.430314:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511450624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878050.430324:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878050.430328:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878050.430331:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878050.430339:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878050.430358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878050.430370:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878050/real 1788878050] req@ffffa061888adc00 x1875771511450752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878050.430437:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511450752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878050.430455:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878050.430462:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878050.430470:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878050.430482:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878050.430518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878050.430535:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878050/real 1788878050] req@ffffa061888ae680 x1875771511450880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878050.430550:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511450880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878050.430560:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878050.430564:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878050.430568:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511450880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878051.069557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878051.069706:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878051.325502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878052.093592:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878052.093672:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878052.349502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878053.117560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878053.117685:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878053.373520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878054.141597:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878054.141626:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878054.397486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878055.165569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878055.165595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878055.421480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878055.549483:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878055.549496:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878055.549502:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878055.549515:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3042 00000100:00080000:2.0:1788878055.549532:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878055.549542:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878055.549595:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511451008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878055.549632:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878055.549637:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878055.549640:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878055.549645:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3042 00000100:00080000:2.0:1788878055.549651:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878055.549656:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878055.549669:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878055.549688:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511451008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878055.549691:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511451136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878055.549714:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878055.549718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878055.549721:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878055.549725:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3042 00000100:00080000:2.0:1788878055.549729:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878055.549733:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878055.549739:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878055.549754:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511451264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878055.549765:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878055.549770:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878055.549773:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878055.549778:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3042 00000100:00080000:2.0:1788878055.549804:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878055.549814:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878055.549816:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878055.549827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878055.549843:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511451392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878055.549855:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878055.549860:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878055.549862:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878055.549867:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3042 00000100:00080000:2.0:1788878055.549871:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878055.549875:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878055.549883:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878055.549901:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878055/real 1788878055] req@ffffa061888af100 x1875771511451008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878055.549911:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511451520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878055.549950:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511451008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878055.549987:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878055.549996:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878055.550005:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511451008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878055.550043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878055.550082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878055.550100:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878055/real 1788878055] req@ffffa061888afb80 x1875771511451136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878055.550117:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511451136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878055.550128:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878055.550132:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878055.550136:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878055.550144:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878055.550164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878055.550176:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878055/real 1788878055] req@ffffa061888ae300 x1875771511451264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878055.550190:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511451264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878055.550200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878055.550203:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878055.550207:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878055.550215:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878055.550235:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878055.550246:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878055/real 1788878055] req@ffffa061888af480 x1875771511451392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878055.550259:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511451392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878055.550269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878055.550272:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878055.550276:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878055.550282:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878055.550299:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878055.550309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878055/real 1788878055] req@ffffa061888ac000 x1875771511451520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878055.550323:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511451520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878055.550332:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878055.550336:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878055.550339:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878056.189526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878056.189598:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878056.445573:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878057.213497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878057.213569:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878057.469481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878058.237479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878058.237615:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878058.493509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878059.261486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878059.261579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878059.517504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878060.285482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878060.285677:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878060.541492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878060.669629:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878060.669642:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878060.669648:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878060.669662:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3047 00000100:00080000:3.0:1788878060.669678:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878060.669707:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878060.669757:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511451648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878060.669823:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878060.669828:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878060.669832:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878060.669840:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3047 00000100:00080000:3.0:1788878060.669846:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878060.669854:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878060.669869:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878060.669903:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511451776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878060.669905:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511451648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878060.669944:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878060.669949:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878060.669952:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878060.669958:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3047 00000100:00080000:3.0:1788878060.669962:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878060.669966:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878060.669974:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878060.669989:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511451904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878060.670002:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878060.670016:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878060.670019:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878060.670030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878060.670032:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3047 00000100:00080000:3.0:1788878060.670042:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878060.670048:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878060.670056:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878060.670070:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511452032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878060.670084:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878060.670089:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878060.670091:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878060.670099:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3047 00000100:00100000:0.0:1788878060.670101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878060/real 1788878060] req@ffffa06188871500 x1875771511451648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878060.670104:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878060.670115:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878060.670128:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878060.670157:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511452160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878060.670163:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511451648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878060.670201:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878060.670207:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878060.670214:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511451648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878060.670245:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878060.670278:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878060.670293:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878060/real 1788878060] req@ffffa06188873100 x1875771511451776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878060.670308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511451776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878060.670319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878060.670323:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878060.670328:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878060.670335:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878060.670354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878060.670365:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878060/real 1788878060] req@ffffa06188872a00 x1875771511451904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878060.670426:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511451904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878060.670442:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878060.670447:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878060.670452:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511451904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878060.670466:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878060.670492:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878060.670505:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878060/real 1788878060] req@ffffa06188873800 x1875771511452032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878060.670519:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511452032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878060.670529:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878060.670532:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878060.670535:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878060.670542:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878060.670562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878060.670572:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878060/real 1788878060] req@ffffa06188873480 x1875771511452160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878060.670586:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511452160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878060.670603:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878060.670607:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878060.670611:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878061.309508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878061.309585:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878061.565466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878062.333463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878062.333569:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878062.589465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878063.357535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878063.357630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878063.613571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878064.381492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878064.381537:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878064.637521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878065.405444:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878065.405520:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878065.661443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878065.789519:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878065.789527:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878065.789531:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878065.789541:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3052 00000100:00080000:3.0:1788878065.789567:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878065.789575:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878065.789611:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511452288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878065.789636:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878065.789639:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878065.789641:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878065.789645:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3052 00000100:00080000:3.0:1788878065.789647:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878065.789651:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878065.789676:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878065.789685:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511452416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878065.789693:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878065.789696:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788878065.789697:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511452288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878065.789698:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878065.789703:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3052 00000100:00080000:3.0:1788878065.789707:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878065.789710:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878065.789713:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878065.789721:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511452544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878065.789726:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878065.789751:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878065.789753:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878065.789756:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3052 00000100:00080000:3.0:1788878065.789760:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878065.789764:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878065.789767:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878065.789776:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511452672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878065.789783:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878065.789787:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788878065.789790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878065.789790:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878065.789796:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3052 00000100:00080000:3.0:1788878065.789799:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878065.789801:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878065.789804:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878065.789812:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511452800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878065.789832:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878065/real 1788878065] req@ffffa06188872300 x1875771511452288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878065.789846:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511452288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878065.789854:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878065.789857:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878065.789860:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511452288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878065.789878:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878065.789894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878065.789902:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878065/real 1788878065] req@ffffa06188873b80 x1875771511452416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878065.789910:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511452416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878065.789915:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878065.789917:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878065.789920:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878065.789924:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878065.789934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878065.789939:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878065/real 1788878065] req@ffffa06188870700 x1875771511452544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878065.789946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511452544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878065.789951:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878065.789953:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878065.789955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878065.789958:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878065.789968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878065.789974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878065/real 1788878065] req@ffffa06188872680 x1875771511452672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878065.789980:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511452672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878065.789985:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878065.789987:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878065.789989:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878065.789992:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878065.790003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878065.790008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878065/real 1788878065] req@ffffa06188872d80 x1875771511452800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878065.790015:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511452800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878065.790020:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878065.790022:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878065.790024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511452800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878066.429498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878066.429596:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878066.685456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878067.453483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878067.453505:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878067.709452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878068.477477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878068.477559:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878068.733455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878069.501521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878069.501703:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878069.757479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878070.525542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878070.525664:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878070.781547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878070.909512:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878070.909523:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878070.909528:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878070.909541:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3057 00000100:00080000:1.0:1788878070.909561:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878070.909572:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878070.909618:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511452928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878070.909641:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878070.909644:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878070.909646:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878070.909650:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3057 00000100:00080000:1.0:1788878070.909653:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878070.909658:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878070.909670:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878070.909687:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511453056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878070.909692:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511452928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878070.909711:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878070.909728:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878070.909730:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878070.909734:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3057 00000100:00080000:1.0:1788878070.909738:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878070.909742:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878070.909747:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878070.909759:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511453184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878070.909767:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878070.909770:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878070.909771:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878070.909774:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3057 00000100:00080000:1.0:1788878070.909777:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878070.909780:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878070.909784:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878070.909785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878070.909795:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511453312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878070.909803:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878070.909806:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878070.909807:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878070.909810:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3057 00000100:00080000:1.0:1788878070.909813:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878070.909816:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878070.909821:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878070.909829:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878070/real 1788878070] req@ffffa0618869b800 x1875771511452928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878070.909830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511453440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878070.909874:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511452928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878070.909884:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878070.909887:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878070.909891:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511452928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878070.909913:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878070.909938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878070.909946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878070/real 1788878070] req@ffffa06188698380 x1875771511453056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878070.909955:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511453056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878070.909967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878070.909970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878070.909972:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878070.909976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878070.909988:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878070.909994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878070/real 1788878070] req@ffffa0618869b100 x1875771511453184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878070.910002:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511453184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878070.910008:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878070.910010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878070.910012:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878070.910016:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878070.910027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878070.910032:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878070/real 1788878070] req@ffffa0618869aa00 x1875771511453312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878070.910039:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511453312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878070.910045:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878070.910047:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878070.910049:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878070.910053:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878070.910063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878070.910069:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878070/real 1788878070] req@ffffa06188699c00 x1875771511453440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878070.910076:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511453440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878070.910082:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878070.910084:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878070.910086:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878071.549464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878071.549551:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878071.805466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878072.573484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878072.573518:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878072.829510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878073.597452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878073.597526:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878073.853491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878074.621457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878074.621508:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878074.877458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878075.645462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878075.645562:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878075.901497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878076.029461:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878076.029470:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878076.029474:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878076.029483:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3062 00000100:00080000:1.0:1788878076.029492:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878076.029499:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878076.029539:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511453568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878076.029559:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878076.029562:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878076.029564:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878076.029568:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3062 00000100:00080000:1.0:1788878076.029570:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878076.029574:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878076.029583:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878076.029588:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511453568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878076.029595:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511453696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878076.029603:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878076.029605:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878076.029607:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878076.029609:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3062 00000100:00080000:1.0:1788878076.029611:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878076.029614:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878076.029618:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878076.029626:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511453824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878076.029632:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878076.029635:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878076.029637:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878076.029639:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3062 00000100:00080000:1.0:1788878076.029641:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878076.029644:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878076.029649:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878076.029655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878076.029664:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511453952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878076.029672:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878076.029675:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878076.029676:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878076.029680:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3062 00000100:00080000:1.0:1788878076.029682:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878076.029687:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878076.029688:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878076/real 1788878076] req@ffffa0618869a680 x1875771511453568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878076.029701:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878076.029713:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511454080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878076.029722:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511453568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878076.029733:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878076.029736:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878076.029740:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511453568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878076.029757:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878076.029774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878076.029783:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878076/real 1788878076] req@ffffa0618869bb80 x1875771511453696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878076.029791:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511453696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878076.029797:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878076.029799:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878076.029801:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878076.029804:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878076.029815:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878076.029821:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878076/real 1788878076] req@ffffa0618869ad80 x1875771511453824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878076.029828:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511453824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878076.029833:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878076.029835:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878076.029836:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878076.029840:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878076.029850:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878076.029856:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878076/real 1788878076] req@ffffa06188698000 x1875771511453952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878076.029863:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511453952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878076.029869:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878076.029871:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878076.029873:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511453952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878076.029876:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878076.029885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878076.029890:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878076/real 1788878076] req@ffffa06188699f80 x1875771511454080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878076.029897:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511454080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878076.029902:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878076.029904:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878076.029906:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878076.669460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878076.669549:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878076.925454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878077.693453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878077.693531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878077.949466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878078.717467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878078.717523:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878078.973498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878079.741459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878079.741570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878079.997458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878080.765486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878080.765554:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878081.021459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878081.149486:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878081.149494:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878081.149499:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878081.149513:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3067 00000100:00080000:2.0:1788878081.149522:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878081.149529:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878081.149561:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511454208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878081.149580:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878081.149583:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878081.149585:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878081.149588:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3067 00000100:00080000:2.0:1788878081.149591:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878081.149594:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878081.149607:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878081.149612:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511454208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878081.149619:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511454336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878081.149628:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878081.149630:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878081.149632:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878081.149634:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3067 00000100:00080000:2.0:1788878081.149636:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878081.149638:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878081.149642:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878081.149652:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511454464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878081.149658:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878081.149661:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878081.149663:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878081.149665:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3067 00000100:00080000:2.0:1788878081.149678:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878081.149684:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878081.149685:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878081.149689:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878081.149700:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511454592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878081.149707:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878081.149709:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878081.149711:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878081.149715:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3067 00000100:00080000:2.0:1788878081.149719:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878081.149723:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878081/real 1788878081] req@ffffa061888adf80 x1875771511454208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878081.149723:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878081.149730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878081.149741:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511454720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878081.149748:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511454208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878081.149759:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878081.149763:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878081.149766:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511454208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878081.149786:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878081.149802:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878081.149810:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878081/real 1788878081] req@ffffa061888ad500 x1875771511454336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878081.149819:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511454336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878081.149824:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878081.149827:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878081.149829:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878081.149832:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878081.149843:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878081.149849:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878081/real 1788878081] req@ffffa061888ac000 x1875771511454464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878081.149856:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511454464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878081.149861:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878081.149863:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878081.149864:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878081.149868:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878081.149879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878081.149885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878081/real 1788878081] req@ffffa061888af480 x1875771511454592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878081.149892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511454592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878081.149898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878081.149899:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878081.149901:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878081.149904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878081.149914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878081.149919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878081/real 1788878081] req@ffffa061888ae300 x1875771511454720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878081.149943:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511454720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878081.149952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878081.149955:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878081.149959:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878081.789447:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878081.789553:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878082.045473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878082.813463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878082.813541:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878083.069443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878083.837489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878083.837564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878084.093465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878084.861482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878084.861535:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878085.117478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878085.885497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878085.885608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878086.141457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878086.269461:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878086.269470:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878086.269474:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878086.269483:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3072 00000100:00080000:2.0:1788878086.269492:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878086.269499:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878086.269546:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511454848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878086.269567:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878086.269569:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878086.269571:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878086.269574:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3072 00000100:00080000:2.0:1788878086.269578:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878086.269582:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878086.269590:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878086.269598:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511454848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878086.269603:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511454976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878086.269612:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878086.269614:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878086.269615:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878086.269618:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3072 00000100:00080000:2.0:1788878086.269620:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878086.269622:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878086.269625:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878086.269634:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511455104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878086.269640:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878086.269642:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788878086.269671:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878086.269700:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878086/real 1788878086] req@ffffa061888afb80 x1875771511454848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878086.269714:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511454848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878086.269727:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878086.269729:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878086.269732:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511454848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878086.269751:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878086.269767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878086.269775:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878086/real 1788878086] req@ffffa061888af100 x1875771511454976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878086.269783:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511454976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878086.269787:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878086.269790:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878086.269795:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788878086.269798:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3072 00000100:00100000:0.0:1788878086.269799:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511454976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878086.269809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455104:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878086.269809:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878086.269818:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878086.269825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878086.269826:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878086.269833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878086/real 1788878086] req@ffffa061888ae680 x1875771511455104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878086.269847:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511455104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878086.269848:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511455232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878086.269861:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878086.269867:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878086.269873:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455104:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878086.269875:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878086.269881:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878086.269884:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878086.269885:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455232:192.168.203.111@tcp:8:kworker.0 00000100:00080000:2.0:1788878086.269891:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3072 00000100:00080000:2.0:1788878086.269898:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878086.269904:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878086.269909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878086.269914:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878086.269925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878086/real 1788878086] req@ffffa061888adc00 x1875771511455232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878086.269932:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511455360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878086.269952:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511455232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878086.269960:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878086.269963:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878086.269965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878086.269976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878086.269991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878086.269999:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878086/real 1788878086] req@ffffa061888aca80 x1875771511455360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878086.270006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511455360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878086.270011:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878086.270013:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878086.270015:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878086.909456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878086.909526:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878087.165468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878087.933512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878087.933573:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878088.189469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878088.957516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878088.957631:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878089.213452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878089.981537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878089.981707:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878090.237467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878091.005448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878091.005692:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878091.261460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878091.389500:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878091.389515:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878091.389520:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878091.389533:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3078 00000100:00080000:2.0:1788878091.389547:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878091.389556:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878091.389599:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511455488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878091.389655:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878091.389659:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878091.389662:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878091.389668:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3078 00000100:00080000:2.0:1788878091.389674:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878091.389679:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878091.389691:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878091.389707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511455616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878091.389721:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878091.389724:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511455488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878091.389726:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878091.389732:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878091.389744:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3078 00000100:00080000:2.0:1788878091.389748:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878091.389751:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878091.389755:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878091.389770:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511455744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878091.389780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878091.389784:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878091.389786:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878091.389790:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3078 00000100:00080000:2.0:1788878091.389804:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878091.389808:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878091.389813:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878091.389818:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878091.389829:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511455872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878091.389854:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878091.389859:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788878091.389863:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878091/real 1788878091] req@ffffa061888ac380 x1875771511455488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878091.389865:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878091.389886:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511455488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878091.389888:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3078 00000100:00080000:2.0:1788878091.389902:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878091.389906:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878091.389913:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788878091.389916:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878091.389936:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511455488:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878091.389937:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878091.389962:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511456000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878091.389973:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878091.390010:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878091.390024:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878091/real 1788878091] req@ffffa061888aea00 x1875771511455616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878091.390039:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511455616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878091.390048:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878091.390061:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878091.390065:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878091.390070:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878091.390085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878091.390094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878091/real 1788878091] req@ffffa061888ad880 x1875771511455744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878091.390104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511455744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878091.390112:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878091.390114:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878091.390117:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878091.390122:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878091.390137:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878091.390146:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878091/real 1788878091] req@ffffa061888aed80 x1875771511455872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878091.390156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511455872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878091.390164:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878091.390166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878091.390169:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511455872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878091.390184:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878091.390198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878091.390207:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878091/real 1788878091] req@ffffa061888ac700 x1875771511456000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878091.390217:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511456000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878091.390224:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878091.390227:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878091.390229:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878092.029490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878092.029612:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878092.285504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878093.053470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878093.053531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878093.309468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878094.077465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878094.077547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878094.333490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878095.101509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878095.101589:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878095.357525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878096.125519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878096.125635:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878096.381500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878096.509493:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878096.509501:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878096.509505:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878096.509515:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3083 00000100:00080000:1.0:1788878096.509524:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878096.509530:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878096.509564:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511456128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878096.509588:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878096.509597:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878096.509610:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878096.509614:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3083 00000100:00080000:1.0:1788878096.509618:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878096.509621:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878096.509629:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878096.509640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511456256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878096.509650:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878096.509650:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511456128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878096.509659:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878096.509661:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878096.509664:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3083 00000100:00080000:1.0:1788878096.509666:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878096.509668:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878096.509672:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878096.509680:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511456384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878096.509687:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878096.509690:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878096.509691:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878096.509694:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3083 00000100:00080000:1.0:1788878096.509697:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878096.509699:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878096.509703:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878096.509710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511456512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878096.509714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878096.509723:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878096.509726:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878096.509727:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878096.509731:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3083 00000100:00080000:1.0:1788878096.509735:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878096.509738:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878096.509749:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878096/real 1788878096] req@ffffa06188699f80 x1875771511456128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878096.509758:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878096.509765:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511456128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878096.509771:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511456640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878096.509804:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878096.509808:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878096.509812:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511456128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878096.509834:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878096.509851:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878096.509859:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878096/real 1788878096] req@ffffa06188698000 x1875771511456256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878096.509867:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511456256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878096.509873:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878096.509875:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878096.509877:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878096.509881:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878096.509891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878096.509897:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878096/real 1788878096] req@ffffa0618869ad80 x1875771511456384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878096.509904:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511456384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878096.509909:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878096.509911:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878096.509913:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878096.509917:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878096.509927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878096.509933:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878096/real 1788878096] req@ffffa0618869bb80 x1875771511456512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878096.509940:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511456512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878096.509945:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878096.509947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878096.509948:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878096.509952:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878096.509961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878096.509966:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878096/real 1788878096] req@ffffa0618869a680 x1875771511456640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878096.509974:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511456640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878096.509978:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878096.509980:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878096.509982:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878097.149490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878097.149624:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878097.405511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878098.173493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878098.173664:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878098.429581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878099.197491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878099.197577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878099.453502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878100.221487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878100.221606:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878100.477542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878101.245490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878101.245621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878101.501490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878101.629542:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878101.629555:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878101.629560:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878101.629574:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3088 00000100:00080000:2.0:1788878101.629601:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878101.629609:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878101.629667:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511456768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878101.629700:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878101.629702:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878101.629704:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878101.629707:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3088 00000100:00080000:2.0:1788878101.629710:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878101.629713:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878101.629722:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878101.629733:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511456896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878101.629742:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878101.629746:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878101.629748:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878101.629755:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3088 00000100:00100000:3.0:1788878101.629757:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511456768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878101.629762:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878101.629771:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878101.629775:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878101.629787:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511457024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878101.629794:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878101.629797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878101.629799:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878101.629803:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3088 00000100:00080000:2.0:1788878101.629816:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878101.629819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878101.629824:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878101.629831:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878101.629839:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511457152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878101.629848:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878101.629850:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878101.629851:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878101.629857:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3088 00000100:00080000:2.0:1788878101.629862:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878101.629866:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878101.629871:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878101.629877:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878101/real 1788878101] req@ffffa061888ace00 x1875771511456768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878101.629884:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511457280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878101.629933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511456768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878101.629944:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878101.629948:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878101.629955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511456768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878101.629982:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878101.629999:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878101.630008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878101/real 1788878101] req@ffffa061888ad180 x1875771511456896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878101.630017:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511456896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878101.630023:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878101.630026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878101.630028:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511456896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878101.630032:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878101.630044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878101.630050:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878101/real 1788878101] req@ffffa061888ac700 x1875771511457024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878101.630058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511457024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878101.630067:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878101.630070:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878101.630072:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878101.630076:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878101.630087:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878101.630093:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878101/real 1788878101] req@ffffa061888aed80 x1875771511457152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878101.630100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511457152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878101.630106:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878101.630108:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878101.630110:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878101.630113:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878101.630124:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878101.630130:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878101/real 1788878101] req@ffffa061888ad880 x1875771511457280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878101.630138:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511457280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878101.630143:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878101.630144:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878101.630147:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878102.269485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878102.269559:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878102.525471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878103.293561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878103.293706:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878103.549481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878104.317524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878104.317525:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878104.573506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878105.341523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878105.341654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878105.597497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878106.365519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878106.365607:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878106.621477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878106.749512:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878106.749535:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878106.749540:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878106.749576:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3093 00000100:00080000:0.0:1788878106.749610:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878106.749623:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878106.749718:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511457408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878106.749774:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878106.749779:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878106.749781:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878106.749786:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3093 00000100:00080000:0.0:1788878106.749792:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878106.749796:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878106.749808:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878106.749841:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511457536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878106.749855:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878106.749855:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511457408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878106.749866:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878106.749870:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878106.749874:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3093 00000100:00080000:0.0:1788878106.749877:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878106.749881:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878106.749886:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878106.749897:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511457664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878106.749912:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878106.749916:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878106.749918:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878106.749921:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3093 00000100:00080000:0.0:1788878106.749925:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878106.749928:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878106.749932:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878106.749944:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511457792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878106.749952:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878106.749956:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878106.749958:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878106.749962:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3093 00000100:00080000:0.0:1788878106.749966:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878106.749970:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878106.749974:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878106.749983:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511457920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878106.749989:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878106.750045:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878106.750089:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878106.750157:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878107.389603:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878107.389677:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878107.645502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878108.413478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878108.413535:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878108.669526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878109.437498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878109.437606:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878109.693478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878109.821646:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878109.821682:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878109.821697:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878109.821735:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878109.821746:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878109.821757:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878109.821768:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788878109.821779:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878106/real 1788878109] req@ffffa06198afd180 x1875771511457536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878109.821810:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511457536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878109.821824:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878109.821829:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878109.821834:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878109.821842:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878106/real 1788878109] req@ffffa06198afd500 x1875771511457664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878109.821854:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511457664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878109.821863:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878109.821867:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878109.821870:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878109.821877:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878106/real 1788878109] req@ffffa06198afca80 x1875771511457792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878109.821890:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511457792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878109.821898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878109.821902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878109.821905:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878109.821911:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878106/real 1788878109] req@ffffa06198afc380 x1875771511457408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878109.821923:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511457408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878109.821930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878109.821933:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878109.821937:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511457408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878109.821976:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878106/real 1788878109] req@ffffa06198afe300 x1875771511457920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878109.821989:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511457920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878109.821996:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878109.822000:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878109.822003:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511457920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878110.461500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878110.461581:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878110.717448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878111.485529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878111.485625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878111.741469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878111.869497:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878111.869505:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878111.869510:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878111.869519:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3098 00000100:00080000:0.0:1788878111.869547:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878111.869554:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878111.869589:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511458048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878111.869611:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878111.869615:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878111.869616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878111.869619:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3098 00000100:00080000:0.0:1788878111.869621:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878111.869624:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878111.869631:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878111.869640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511458176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878111.869648:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878111.869651:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878111.869652:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878111.869655:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3098 00000100:00080000:0.0:1788878111.869657:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878111.869659:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878111.869662:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878111.869671:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511458304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878111.869673:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511458048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878111.869683:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878111.869687:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878111.869689:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878111.869691:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3098 00000100:00080000:0.0:1788878111.869693:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878111.869696:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878111.869699:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878111.869709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511458432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878111.869715:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878111.869717:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878111.869719:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878111.869722:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3098 00000100:00080000:0.0:1788878111.869725:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878111.869727:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878111.869730:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878111.869755:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511458560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878111.869767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878111.869835:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878111/real 1788878111] req@ffffa06198afea00 x1875771511458048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878111.869857:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511458048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878111.869876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878111.869899:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878111.869903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511458048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878111.869912:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878111.869936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878111.869949:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878111/real 1788878111] req@ffffa06198afc000 x1875771511458176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878111.869962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511458176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878111.869971:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878111.869975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878111.869978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878111.870003:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878111.870025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878111.870037:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878111/real 1788878111] req@ffffa06198afc700 x1875771511458304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878111.870050:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511458304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878111.870059:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878111.870062:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878111.870066:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878111.870085:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878111.870112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878111.870125:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878111/real 1788878111] req@ffffa06198afdc00 x1875771511458432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878111.870138:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511458432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878111.870146:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878111.870150:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878111.870153:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878111.870160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878111.870179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878111.870189:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878111/real 1788878111] req@ffffa06198afdf80 x1875771511458560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878111.870201:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511458560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878111.870216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878111.870219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878111.870223:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878112.509487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878112.509542:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878112.765494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878113.533665:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878113.533820:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878113.789504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878114.557507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878114.557604:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878114.813461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878115.581623:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878115.581709:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878115.837545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878116.605578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878116.605643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878116.861562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878116.989555:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878116.989566:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878116.989572:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878116.989585:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3103 00000100:00080000:0.0:1788878116.989597:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878116.989606:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878116.989662:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511458688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878116.989694:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878116.989699:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878116.989701:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878116.989705:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3103 00000100:00080000:0.0:1788878116.989709:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878116.989713:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878116.989724:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878116.989739:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511458816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878116.989750:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878116.989755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878116.989757:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878116.989761:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3103 00000100:00080000:0.0:1788878116.989764:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878116.989773:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878116.989774:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511458688:192.168.203.111@tcp:250:kworker.0 00000008:00100000:0.0:1788878116.989778:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878116.989796:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511458944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878116.989815:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878116.989819:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878116.989821:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878116.989825:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3103 00000100:00080000:0.0:1788878116.989828:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878116.989832:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878116.989837:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878116.989851:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511459072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878116.989861:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878116.989864:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878116.989866:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878116.989870:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3103 00000100:00080000:0.0:1788878116.989876:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788878116.989879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878116.989884:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878116.989891:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878116.989905:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511459200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878116.989973:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878116/real 1788878116] req@ffffa06198aff800 x1875771511458688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878116.990027:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511458688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878116.990041:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878116.990047:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878116.990053:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511458688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878116.990063:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878116.990095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878116.990116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878116/real 1788878116] req@ffffa06198affb80 x1875771511458816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878116.990134:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511458816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878116.990149:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878116.990155:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878116.990161:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878116.990197:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878116.990234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878116.990250:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878116/real 1788878116] req@ffffa06198afd880 x1875771511458944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878116.990264:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511458944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878116.990273:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878116.990276:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878116.990280:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511458944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878116.990286:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878116.990305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878116.990315:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878116/real 1788878116] req@ffffa06198aff480 x1875771511459072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878116.990327:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511459072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878116.990336:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878116.990338:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878116.990342:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878116.990347:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878116.990362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878116.990371:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878116/real 1788878116] req@ffffa06198afed80 x1875771511459200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878116.990459:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511459200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878116.990470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878116.990473:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878116.990477:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878117.629565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878117.629640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878117.885557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878118.653486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878118.653568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878118.909490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878119.677448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878119.677603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878119.933460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878120.701540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878120.701679:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878120.957473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878121.725473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878121.725607:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878121.981503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878122.109473:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878122.109497:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878122.109503:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878122.109529:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3108 00000100:00080000:0.0:1788878122.109537:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878122.109543:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878122.109579:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511459328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878122.109608:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878122.109611:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878122.109613:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878122.109617:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3108 00000100:00080000:0.0:1788878122.109622:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878122.109627:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878122.109637:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878122.109644:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511459328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878122.109656:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511459456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878122.109669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878122.109673:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878122.109674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878122.109677:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3108 00000100:00080000:0.0:1788878122.109681:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878122.109684:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878122.109689:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878122.109702:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511459584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878122.109702:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878122.109712:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878122.109715:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878122.109717:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878122.109720:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3108 00000100:00080000:0.0:1788878122.109722:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878122.109725:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878122.109728:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878122.109737:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878122/real 1788878122] req@ffffa06198afed80 x1875771511459328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878122.109738:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511459712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878122.109766:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878122.109769:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878122.109770:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878122.109772:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511459328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878122.109776:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3108 00000100:00080000:0.0:1788878122.109781:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878122.109788:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788878122.109790:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1788878122.109792:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788878122.109796:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878122.109800:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511459328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878122.109803:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511459840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878122.109846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878122.109895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878122.109937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878122/real 1788878122] req@ffffa06198aff480 x1875771511459456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878122.109952:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511459456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878122.109967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878122.109970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878122.109972:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878122.109976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878122.109995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878122.110006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878122/real 1788878122] req@ffffa06198afd880 x1875771511459584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878122.110019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511459584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878122.110027:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878122.110031:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878122.110034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878122.110040:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878122.110062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878122.110074:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878122/real 1788878122] req@ffffa06198affb80 x1875771511459712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878122.110086:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511459712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878122.110092:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878122.110094:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878122.110097:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878122.110101:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878122.110114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878122.110121:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878122/real 1788878122] req@ffffa06198aff800 x1875771511459840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878122.110129:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511459840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878122.110134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878122.110135:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878122.110138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511459840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878122.749534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878122.749612:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878123.005531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878123.773556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878123.773678:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878124.029526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878124.797566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878124.797589:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878125.053593:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878125.821569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878125.821633:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878126.077533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878126.845553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878126.845650:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878127.101484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878127.229524:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878127.229532:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878127.229537:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878127.229546:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3113 00000100:00080000:1.0:1788878127.229555:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878127.229562:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878127.229594:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511459968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878127.229615:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878127.229618:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878127.229620:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878127.229623:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3113 00000100:00080000:1.0:1788878127.229625:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878127.229627:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878127.229634:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878127.229643:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511460096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878127.229656:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878127.229660:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788878127.229660:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511459968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878127.229662:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878127.229668:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3113 00000100:00080000:1.0:1788878127.229671:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878127.229673:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878127.229676:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878127.229684:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511460224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878127.229691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878127.229693:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878127.229695:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878127.229697:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3113 00000100:00080000:1.0:1788878127.229699:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878127.229702:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878127.229704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878127.229711:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511460352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878127.229716:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878127.229720:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1788878127.229721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878127.229722:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878127.229728:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3113 00000100:00080000:1.0:1788878127.229731:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878127.229735:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878127.229738:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878127.229745:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511460480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878127.229755:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878127/real 1788878127] req@ffffa06188699c00 x1875771511459968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878127.229772:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511459968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878127.229784:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878127.229788:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878127.229793:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511459968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878127.229815:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878127.229840:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878127.229855:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878127/real 1788878127] req@ffffa0618869aa00 x1875771511460096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878127.229870:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511460096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878127.229879:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878127.229882:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878127.229884:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878127.229889:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878127.229904:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878127.229911:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878127/real 1788878127] req@ffffa0618869b100 x1875771511460224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878127.229919:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511460224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878127.229930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878127.229932:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878127.229934:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878127.229938:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878127.229950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878127.229956:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878127/real 1788878127] req@ffffa06188698380 x1875771511460352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878127.229963:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511460352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878127.229969:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878127.229971:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878127.229973:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878127.229976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878127.229986:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878127.229995:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878127/real 1788878127] req@ffffa0618869b800 x1875771511460480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878127.230007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511460480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878127.230016:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878127.230019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878127.230023:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878127.869541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878127.869603:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878128.125533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878128.893546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878128.893661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878129.149564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878129.917600:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878129.917702:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878130.173533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878130.941561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878130.941694:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878131.197601:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878131.965582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878131.965652:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878132.221508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878132.349555:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878132.349571:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878132.349577:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878132.349591:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3119 00000100:00080000:1.0:1788878132.349606:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878132.349617:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878132.349667:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511460608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878132.349707:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878132.349711:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878132.349714:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878132.349719:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3119 00000100:00080000:1.0:1788878132.349724:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878132.349728:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878132.349740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878132.349755:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511460736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878132.349770:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878132.349775:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878132.349777:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878132.349782:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3119 00000100:00080000:1.0:1788878132.349786:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878132.349790:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878132.349798:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878132.349801:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511460608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878132.349816:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511460864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878132.349831:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878132.349836:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878132.349838:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878132.349844:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3119 00000100:00080000:1.0:1788878132.349849:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878132.349853:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878132.349860:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878132.349877:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511460992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878132.349890:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788878132.349896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878132.349899:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878132.349906:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878132.349914:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3119 00000100:00080000:1.0:1788878132.349921:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878132.349939:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878132.349946:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878132.349963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878132/real 1788878132] req@ffffa0618869b800 x1875771511460608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878132.349973:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511461120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878132.350029:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511460608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878132.350051:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878132.350056:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878132.350062:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511460608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878132.350074:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878132.350109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878132.350127:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878132/real 1788878132] req@ffffa06188698380 x1875771511460736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878132.350143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511460736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878132.350154:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878132.350158:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878132.350163:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878132.350193:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878132.350224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878132.350247:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878132/real 1788878132] req@ffffa0618869b100 x1875771511460864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878132.350270:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511460864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878132.350287:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878132.350295:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878132.350301:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878132.350313:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878132.350351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878132.350374:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878132/real 1788878132] req@ffffa0618869aa00 x1875771511460992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878132.350476:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511460992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878132.350488:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878132.350492:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878132.350496:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511460992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878132.350505:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878132.350535:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878132.350551:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878132/real 1788878132] req@ffffa06188699c00 x1875771511461120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878132.350565:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511461120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878132.350575:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878132.350579:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878132.350583:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878132.989543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878132.989589:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878133.245542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878134.013630:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878134.013721:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878134.269594:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788878135.037618:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788878135.037694:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788878135.293538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878136.061487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878136.061536:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878136.317467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878137.085502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878137.085585:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878137.341510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878137.469484:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878137.469495:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878137.469500:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878137.469512:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3124 00000100:00080000:1.0:1788878137.469523:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878137.469532:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878137.469570:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511461248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878137.469598:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878137.469603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878137.469605:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878137.469610:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3124 00000100:00080000:1.0:1788878137.469614:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878137.469620:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878137.469630:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878137.469639:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511461248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878137.469646:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511461376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878137.469660:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878137.469664:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878137.469666:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878137.469671:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3124 00000100:00080000:1.0:1788878137.469675:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878137.469679:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878137.469685:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788878137.469696:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878137.469700:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511461504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878137.469712:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878137.469716:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878137.469718:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878137.469723:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3124 00000100:00100000:3.0:1788878137.469727:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878137/real 1788878137] req@ffffa0618869a680 x1875771511461248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878137.469730:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878137.469739:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878137.469747:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878137.469748:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511461248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878137.469764:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511461632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878137.469765:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878137.469772:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878137.469779:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511461248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878137.469788:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878137.469793:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878137.469795:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878137.469802:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3124 00000100:00100000:3.0:1788878137.469803:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461376:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788878137.469810:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878137.469818:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878137.469824:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878137.469825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878137.469836:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878137/real 1788878137] req@ffffa0618869bb80 x1875771511461376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878137.469844:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511461760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878137.469850:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511461376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878137.469872:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878137.469875:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878137.469878:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878137.469882:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878137.469895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878137.469901:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878137/real 1788878137] req@ffffa0618869ad80 x1875771511461504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878137.469917:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511461504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878137.469923:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878137.469925:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878137.469927:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878137.469932:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878137.469943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878137.469954:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878137/real 1788878137] req@ffffa06188698000 x1875771511461632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878137.469962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511461632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878137.469967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878137.469969:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878137.469971:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878137.469981:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878137.469992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878137.469997:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878137/real 1788878137] req@ffffa06188699f80 x1875771511461760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878137.470004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511461760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878137.470009:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878137.470011:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878137.470013:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511461760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878138.109468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878138.109525:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878138.365476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878139.133547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878139.133621:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878139.389549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878140.157569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878140.157662:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878140.413492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878141.181472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878141.181527:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878141.437495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878142.205483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878142.205542:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878142.461514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878142.589493:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878142.589505:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878142.589509:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878142.589521:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3129 00000100:00080000:1.0:1788878142.589532:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878142.589541:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878142.589581:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511461888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878142.589611:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878142.589614:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878142.589615:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878142.589618:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3129 00000100:00080000:1.0:1788878142.589622:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878142.589626:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878142.589635:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878142.589648:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511461888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878142.589653:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511462016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878142.589668:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878142.589672:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878142.589674:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878142.589678:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3129 00000100:00080000:1.0:1788878142.589683:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878142.589687:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878142.589692:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878142.589703:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511462144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878142.589711:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878142.589718:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878142.589721:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878142.589723:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878142.589727:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3129 00000100:00080000:1.0:1788878142.589731:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878142.589734:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878142.589738:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878142.589746:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878142/real 1788878142] req@ffffa06188699f80 x1875771511461888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878142.589748:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511462272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878142.589772:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878142.589775:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511461888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878142.589779:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878142.589784:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878142.589789:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878142.589793:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3129 00000100:00080000:3.0:1788878142.589797:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788878142.589801:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878142.589802:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511461888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878142.589811:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878142.589817:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878142.589825:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788878142.589830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511462400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878142.589847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878142.589858:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878142/real 1788878142] req@ffffa06188698000 x1875771511462016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878142.589866:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511462016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878142.589872:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878142.589874:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878142.589877:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878142.589880:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878142.589891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878142.589896:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878142/real 1788878142] req@ffffa0618869ad80 x1875771511462144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878142.589904:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511462144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878142.589924:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878142.589926:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878142.589928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878142.589931:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878142.589941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878142.589946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878142/real 1788878142] req@ffffa0618869bb80 x1875771511462272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878142.589953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511462272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878142.589959:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878142.589961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878142.589963:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878142.589973:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878142.589982:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878142.590000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878142/real 1788878142] req@ffffa0618869a680 x1875771511462400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878142.590013:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511462400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878142.590022:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878142.590026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878142.590035:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878143.229481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878143.229575:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878143.485535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878144.253515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878144.253727:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878144.509491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878145.277467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878145.277530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878145.533476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878146.301571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878146.301646:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878146.557497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878147.325459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878147.325545:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878147.581477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878147.709517:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878147.709529:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878147.709534:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878147.709547:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3134 00000100:00080000:2.0:1788878147.709560:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878147.709569:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878147.709621:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511462528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878147.709644:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878147.709647:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878147.709649:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878147.709652:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3134 00000100:00080000:2.0:1788878147.709656:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878147.709658:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878147.709668:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878147.709677:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511462656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878147.709684:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878147.709687:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878147.709688:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878147.709691:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3134 00000100:00080000:2.0:1788878147.709693:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878147.709695:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878147.709699:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878147.709707:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511462784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878147.709714:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878147.709718:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878147.709719:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878147.709722:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3134 00000100:00080000:2.0:1788878147.709725:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878147.709727:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511462528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878147.709730:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878147.709737:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878147.709744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511462912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878147.709750:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878147.709753:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878147.709754:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878147.709757:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3134 00000100:00080000:2.0:1788878147.709759:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878147.709761:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878147.709764:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878147.709773:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511463040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878147.709810:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878147.709849:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878147/real 1788878147] req@ffffa061888aea00 x1875771511462528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878147.709865:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511462528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878147.709875:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878147.709878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878147.709882:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511462528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878147.709898:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878147.709913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878147.709921:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878147/real 1788878147] req@ffffa061888ac380 x1875771511462656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878147.709929:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511462656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878147.709934:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878147.709936:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878147.709938:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878147.709972:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878147.709983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878147.710001:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878147/real 1788878147] req@ffffa061888aca80 x1875771511462784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878147.710010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511462784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878147.710016:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878147.710018:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878147.710020:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878147.710024:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878147.710036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878147.710042:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878147/real 1788878147] req@ffffa061888adc00 x1875771511462912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878147.710060:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511462912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878147.710065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878147.710067:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878147.710069:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511462912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878147.710072:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878147.710083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878147.710089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878147/real 1788878147] req@ffffa061888ae680 x1875771511463040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878147.710096:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511463040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878147.710102:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878147.710104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878147.710106:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878148.349524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878148.349603:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878148.605511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878149.373523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878149.373568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878149.629531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878150.397528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878150.397612:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878150.653532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878151.421580:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878151.421655:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878151.677503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878152.445489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878152.445617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878152.701495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878152.829492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878152.829500:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878152.829504:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878152.829512:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3139 00000100:00080000:1.0:1788878152.829521:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878152.829527:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878152.829559:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511463168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878152.829595:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878152.829599:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878152.829601:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878152.829605:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3139 00000100:00080000:1.0:1788878152.829610:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878152.829616:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878152.829624:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878152.829634:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511463168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878152.829639:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511463296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878152.829648:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878152.829651:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878152.829652:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878152.829655:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3139 00000100:00080000:1.0:1788878152.829657:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878152.829660:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878152.829665:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878152.829678:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511463424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878152.829689:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788878152.829691:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878152.829695:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878152.829699:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878152.829705:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3139 00000100:00080000:1.0:1788878152.829710:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878152.829715:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878152.829720:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878152.829723:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878152/real 1788878152] req@ffffa06188699c00 x1875771511463168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878152.829733:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511463552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878152.829741:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511463168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878152.829743:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878152.829750:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878152.829754:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878152.829762:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3139 00000100:00080000:3.0:1788878152.829763:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878152.829768:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878152.829771:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788878152.829775:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878152.829779:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511463168:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878152.829780:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878152.829792:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511463680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878152.829803:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878152.829822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878152.829830:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878152/real 1788878152] req@ffffa0618869aa00 x1875771511463296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878152.829838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511463296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878152.829845:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878152.829847:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878152.829849:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878152.829853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878152.829864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878152.829870:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878152/real 1788878152] req@ffffa0618869b100 x1875771511463424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878152.829878:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511463424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878152.829883:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878152.829885:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878152.829886:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878152.829890:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878152.829901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878152.829906:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878152/real 1788878152] req@ffffa06188698380 x1875771511463552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878152.829928:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511463552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878152.829936:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878152.829940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878152.829943:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878152.829950:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878152.829971:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878152.829983:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878152/real 1788878152] req@ffffa0618869b800 x1875771511463680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878152.829996:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511463680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878152.830005:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878152.830009:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878152.830012:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878153.469437:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878153.469506:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878153.725472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878154.493520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878154.493649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878154.749498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878155.517530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878155.517640:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878155.773495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878156.541525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878156.541608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878156.797494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878157.565467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878157.565613:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878157.821593:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878157.949478:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878157.949494:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878157.949500:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878157.949515:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3144 00000100:00080000:1.0:1788878157.949530:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878157.949541:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878157.949590:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511463808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878157.949629:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878157.949632:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878157.949634:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878157.949637:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3144 00000100:00080000:1.0:1788878157.949641:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878157.949643:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878157.949653:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878157.949664:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511463936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878157.949684:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878157.949687:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878157.949688:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878157.949690:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3144 00000100:00080000:1.0:1788878157.949692:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878157.949697:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878157.949698:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511463808:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878157.949700:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878157.949711:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511464064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878157.949719:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878157.949723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878157.949725:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878157.949729:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3144 00000100:00080000:1.0:1788878157.949733:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878157.949737:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878157.949744:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878157.949759:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511464192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878157.949770:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878157.949773:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878157.949774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878157.949778:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3144 00000800:00000100:3.0:1788878157.949784:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878157.949785:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878157.949792:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878157.949796:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878157.949807:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511464320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878157.949839:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878157/real 1788878157] req@ffffa0618869b800 x1875771511463808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878157.949858:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511463808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878157.949868:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878157.949871:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878157.949875:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511463808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878157.949880:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878157.949899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878157.949908:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878157/real 1788878157] req@ffffa06188698380 x1875771511463936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878157.949916:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511463936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878157.949928:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878157.949930:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878157.949933:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511463936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878157.949958:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878157.949969:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878157.949975:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878157/real 1788878157] req@ffffa0618869b100 x1875771511464064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878157.949982:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511464064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878157.949997:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878157.949999:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878157.950001:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878157.950005:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878157.950016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878157.950021:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878157/real 1788878157] req@ffffa0618869aa00 x1875771511464192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878157.950029:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511464192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878157.950034:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878157.950036:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878157.950039:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878157.950043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878157.950063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878157.950068:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878157/real 1788878157] req@ffffa06188699c00 x1875771511464320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878157.950075:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511464320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878157.950081:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878157.950082:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878157.950084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878158.589506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878158.589647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878158.845543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878159.613484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878159.613543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878159.869449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878160.637500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878160.637544:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878160.893507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878161.661571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878161.661642:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878161.917458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878162.685475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878162.685566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878162.941477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878163.069498:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878163.069508:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878163.069512:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878163.069521:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3149 00000100:00080000:2.0:1788878163.069531:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878163.069537:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878163.069569:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511464448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878163.069590:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878163.069593:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878163.069595:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878163.069598:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3149 00000100:00080000:2.0:1788878163.069600:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878163.069602:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878163.069610:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878163.069622:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511464576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878163.069630:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878163.069634:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788878163.069635:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511464448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878163.069636:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878163.069644:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3149 00000100:00080000:2.0:1788878163.069647:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878163.069649:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878163.069652:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878163.069662:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511464704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878163.069669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878163.069671:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878163.069672:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878163.069676:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3149 00000100:00080000:2.0:1788878163.069686:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878163.069688:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878163.069691:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878163.069697:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878163.069701:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511464832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878163.069709:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878163.069712:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878163.069713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878163.069717:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3149 00000100:00080000:2.0:1788878163.069720:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878163.069722:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878163.069726:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878163.069730:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878163/real 1788878163] req@ffffa061888af100 x1875771511464448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878163.069735:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511464960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878163.069755:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511464448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878163.069765:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878163.069768:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878163.069771:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511464448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878163.069788:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878163.069802:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878163.069811:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878163/real 1788878163] req@ffffa061888afb80 x1875771511464576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878163.069820:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511464576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878163.069826:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878163.069829:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878163.069831:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878163.069835:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878163.069845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878163.069850:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878163/real 1788878163] req@ffffa061888ae300 x1875771511464704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878163.069858:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511464704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878163.069864:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878163.069866:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878163.069867:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878163.069871:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878163.069880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878163.069886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878163/real 1788878163] req@ffffa061888af480 x1875771511464832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878163.069893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511464832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878163.069899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878163.069901:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878163.069903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878163.069906:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878163.069922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878163.069948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878163/real 1788878163] req@ffffa061888ac000 x1875771511464960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878163.069961:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511464960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878163.069970:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878163.069972:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878163.069975:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511464960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878163.709487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878163.709558:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878163.965462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878164.733465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878164.733528:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878164.989457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878165.757453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878165.757525:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878166.013502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878166.781481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878166.781584:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878167.037496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878167.805486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878167.805580:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878168.061592:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878168.189534:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878168.189545:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878168.189550:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878168.189559:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3154 00000100:00080000:2.0:1788878168.189571:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878168.189580:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878168.189626:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511465088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878168.189653:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878168.189656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878168.189657:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878168.189661:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3154 00000100:00080000:2.0:1788878168.189663:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878168.189666:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878168.189674:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878168.189683:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511465216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878168.189691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878168.189693:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878168.189695:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878168.189697:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3154 00000100:00080000:2.0:1788878168.189699:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878168.189701:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878168.189705:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878168.189709:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511465088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878168.189716:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511465344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878168.189725:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878168.189727:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878168.189728:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878168.189731:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3154 00000100:00080000:2.0:1788878168.189733:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878168.189735:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878168.189738:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878168.189745:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511465472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878168.189752:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878168.189754:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878168.189756:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878168.189759:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3154 00000100:00080000:2.0:1788878168.189769:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878168.189772:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878168.189775:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878168.189781:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878168.189785:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511465600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878168.189818:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878168/real 1788878168] req@ffffa061888ad500 x1875771511465088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878168.189833:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511465088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878168.189841:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878168.189844:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878168.189847:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511465088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878168.189853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878168.189875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878168.189884:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878168/real 1788878168] req@ffffa061888adf80 x1875771511465216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878168.189892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511465216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878168.189898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878168.189901:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878168.189904:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878168.189929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878168.189969:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878168.189982:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878168/real 1788878168] req@ffffa061888ac000 x1875771511465344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878168.189995:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511465344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878168.190005:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878168.190008:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878168.190012:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878168.190019:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878168.190039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878168.190048:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878168/real 1788878168] req@ffffa061888af480 x1875771511465472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878168.190074:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511465472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878168.190080:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878168.190083:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878168.190085:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878168.190090:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878168.190110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878168.190122:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878168/real 1788878168] req@ffffa061888ae300 x1875771511465600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878168.190137:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511465600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878168.190147:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878168.190152:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878168.190154:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878168.829508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878168.829567:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878169.085453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878169.853506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878169.853570:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878170.109460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878170.877461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878170.877539:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878171.133518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878171.901537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878171.901690:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878172.157493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878172.925484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878172.925573:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878173.181488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878173.309466:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878173.309474:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878173.309478:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878173.309488:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3160 00000100:00080000:2.0:1788878173.309510:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878173.309517:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878173.309554:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511465728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878173.309595:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878173.309599:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878173.309600:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878173.309604:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3160 00000100:00080000:2.0:1788878173.309606:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878173.309610:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878173.309618:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878173.309628:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511465856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878173.309645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878173.309647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878173.309648:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878173.309650:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3160 00000100:00080000:2.0:1788878173.309652:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878173.309654:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878173.309657:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878173.309662:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511465984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878173.309668:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878173.309670:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878173.309671:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878173.309674:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3160 00000100:00080000:2.0:1788878173.309677:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878173.309680:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878173.309683:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878173.309690:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511466112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878173.309698:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878173.309700:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878173.309702:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878173.309725:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3160 00000100:00080000:2.0:1788878173.309732:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878173.309736:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511465728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878173.309737:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878173.309743:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878173.309757:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511466240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878173.309891:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878173.309914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878173.309940:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878173.309958:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878173.949527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878173.949628:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878174.205506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878174.973457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878174.973541:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878175.229466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878175.997502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878175.997579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878176.253513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878176.381985:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878176.382052:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878176.382091:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878176.382150:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878176.382164:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878176.382176:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878176.382190:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788878176.382280:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878173/real 1788878176] req@ffffa061888adc00 x1875771511466112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878176.382345:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511466112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878176.382362:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878176.382369:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878176.382376:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878176.382447:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878173/real 1788878176] req@ffffa061888aca80 x1875771511466240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878176.382476:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511466240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878176.382489:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878176.382495:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878176.382500:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878176.382510:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878173/real 1788878176] req@ffffa061888afb80 x1875771511465728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878176.382527:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511465728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878176.382539:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878176.382544:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878176.382549:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511465728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878176.382560:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878173/real 1788878176] req@ffffa061888af100 x1875771511465856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878176.382577:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511465856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878176.382588:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878176.382594:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878176.382599:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878176.382610:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878173/real 1788878176] req@ffffa061888ae680 x1875771511465984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878176.382625:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511465984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878176.382637:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878176.382642:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878176.382647:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511465984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878177.021483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878177.021592:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878177.277475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878178.045481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878178.045529:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878178.301492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878178.429455:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878178.429464:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878178.429468:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878178.429484:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3165 00000100:00080000:2.0:1788878178.429494:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878178.429502:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878178.429531:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511466368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878178.429556:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878178.429559:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878178.429561:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878178.429564:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3165 00000100:00080000:2.0:1788878178.429567:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878178.429570:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878178.429578:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878178.429598:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511466496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878178.429606:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878178.429609:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878178.429610:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878178.429613:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3165 00000100:00080000:2.0:1788878178.429615:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878178.429618:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511466368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878178.429620:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878178.429624:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878178.429640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511466624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878178.429652:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878178.429656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878178.429658:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878178.429663:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3165 00000100:00080000:2.0:1788878178.429675:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878178.429680:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788878178.429686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878178.429689:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878178.429702:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511466752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878178.429726:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878178.429728:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878178.429730:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878178.429735:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3165 00000100:00100000:3.0:1788878178.429738:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878178/real 1788878178] req@ffffa061888ac380 x1875771511466368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878178.429739:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878178.429745:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878178.429750:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878178.429762:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511466880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878178.429779:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511466368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878178.429792:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878178.429797:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878178.429801:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511466368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878178.429809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878178.429833:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878178.429847:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878178/real 1788878178] req@ffffa061888aea00 x1875771511466496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878178.429872:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511466496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878178.429882:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878178.429884:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878178.429888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878178.429913:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878178.429936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878178.429949:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878178/real 1788878178] req@ffffa061888ad880 x1875771511466624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878178.429961:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511466624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878178.429974:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878178.429977:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878178.429980:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878178.429986:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878178.430006:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878178.430018:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878178/real 1788878178] req@ffffa061888aed80 x1875771511466752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878178.430030:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511466752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878178.430038:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878178.430041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878178.430044:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878178.430050:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878178.430074:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878178.430087:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878178/real 1788878178] req@ffffa061888ac700 x1875771511466880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878178.430098:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511466880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878178.430105:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878178.430109:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878178.430113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511466880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878179.069559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878179.069636:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878179.325527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878180.093522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878180.093595:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878180.349525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878181.117543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878181.117622:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878181.373502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878182.141484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878182.141629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878182.397468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878183.165533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878183.165701:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878183.421502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878183.549531:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878183.549549:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878183.549557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878183.549576:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3170 00000100:00080000:2.0:1788878183.549593:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878183.549604:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878183.549649:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511467008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878183.549685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878183.549689:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878183.549693:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878183.549698:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3170 00000100:00080000:2.0:1788878183.549703:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878183.549707:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878183.549719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878183.549733:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511467136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878183.549744:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878183.549748:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878183.549750:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878183.549754:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3170 00000100:00080000:2.0:1788878183.549757:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878183.549761:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878183.549767:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878183.549781:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511467264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878183.549795:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878183.549800:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878183.549806:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878183.549809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511467008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878183.549814:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3170 00000100:00080000:2.0:1788878183.549823:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878183.549829:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878183.549835:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878183.549853:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511467392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878183.549864:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878183.549868:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878183.549871:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878183.549876:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3170 00000100:00080000:2.0:1788878183.549892:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878183.549897:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878183.549903:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878183.549912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878183.549921:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511467520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878183.549999:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878183/real 1788878183] req@ffffa061888ad180 x1875771511467008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878183.550049:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511467008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878183.550068:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878183.550073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878183.550079:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511467008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878183.550089:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878183.550127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878183.550152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878183/real 1788878183] req@ffffa061888ace00 x1875771511467136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878183.550175:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511467136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878183.550192:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878183.550198:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878183.550204:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878183.550296:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878183.550363:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878183.550436:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878183/real 1788878183] req@ffffa061888ac700 x1875771511467264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878183.550458:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511467264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878183.550484:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878183.550508:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878183.550514:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878183.550525:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878183.550566:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878183.550588:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878183/real 1788878183] req@ffffa061888aed80 x1875771511467392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878183.550610:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511467392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878183.550620:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878183.550624:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878183.550628:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878183.550635:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878183.550661:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878183.550674:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878183/real 1788878183] req@ffffa061888ad880 x1875771511467520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878183.550687:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511467520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878183.550696:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878183.550699:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878183.550704:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878184.189461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878184.189515:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878184.445563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878185.213539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878185.213687:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878185.469526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878186.237480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878186.237542:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878186.493492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878187.261473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878187.261544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878187.517523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878188.285506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878188.285595:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878188.541505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878188.669535:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878188.669550:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878188.669557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878188.669573:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3175 00000100:00080000:2.0:1788878188.669588:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878188.669601:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878188.669651:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511467648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878188.669689:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878188.669698:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878188.669701:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878188.669709:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3175 00000100:00080000:2.0:1788878188.669717:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878188.669725:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878188.669741:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878188.669784:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511467648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878188.669791:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511467776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878188.669814:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878188.669818:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878188.669821:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878188.669826:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3175 00000100:00080000:2.0:1788878188.669832:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878188.669838:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878188.669847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878188.669867:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511467904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878188.669879:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878188.669884:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878188.669886:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878188.669895:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3175 00000800:00000100:3.0:1788878188.669896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878188.669902:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878188.669909:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878188.669918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878188.669934:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511468032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878188.669946:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878188.669954:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878188/real 1788878188] req@ffffa061888aea00 x1875771511467648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878188.669955:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878188.669961:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878188.669976:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3175 00000100:00080000:2.0:1788878188.669985:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878188.669992:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878188.669999:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878188.670002:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511467648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878188.670016:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511468160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878188.670023:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878188.670039:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878188.670049:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511467648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878188.670080:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878188.670109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878188.670125:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878188/real 1788878188] req@ffffa061888ac380 x1875771511467776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878188.670141:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511467776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878188.670152:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878188.670156:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878188.670160:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878188.670167:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878188.670228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878188.670253:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878188/real 1788878188] req@ffffa061888ae680 x1875771511467904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878188.670279:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511467904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878188.670297:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878188.670304:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878188.670311:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511467904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878188.670324:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878188.670359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878188.670375:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878188/real 1788878188] req@ffffa061888af100 x1875771511468032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878188.670438:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511468032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878188.670449:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878188.670453:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878188.670457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878188.670465:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878188.670490:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878188.670504:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878188/real 1788878188] req@ffffa061888afb80 x1875771511468160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878188.670518:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511468160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878188.670529:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878188.670532:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878188.670536:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878189.309475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878189.309538:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878189.565470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878190.333459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878190.333571:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878190.589497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878191.357560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878191.357651:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878191.613499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878192.381455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878192.381536:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878192.637515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878193.405487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878193.405579:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878193.661485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878193.789547:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878193.789556:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878193.789560:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878193.789569:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3180 00000100:00080000:1.0:1788878193.789580:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878193.789586:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878193.789621:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511468288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878193.789648:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878193.789651:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878193.789653:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878193.789656:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3180 00000100:00080000:1.0:1788878193.789659:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878193.789661:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878193.789671:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878193.789681:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511468416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878193.789691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878193.789696:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511468288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878193.789697:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878193.789701:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878193.789709:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3180 00000100:00080000:1.0:1788878193.789712:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878193.789715:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878193.789719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878193.789730:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511468544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878193.789737:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878193.789741:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878193.789742:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878193.789746:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3180 00000100:00080000:1.0:1788878193.789749:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878193.789752:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878193.789756:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878193.789764:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511468672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878193.789769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878193.789779:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878193.789783:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878193.789785:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878193.789789:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3180 00000100:00080000:1.0:1788878193.789792:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878193.789795:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878193.789799:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878193.789809:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878193/real 1788878193] req@ffffa0618869a680 x1875771511468288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878193.789810:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511468800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878193.789850:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511468288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878193.789861:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878193.789864:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878193.789867:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511468288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878193.789887:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878193.789905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878193.789918:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878193/real 1788878193] req@ffffa0618869bb80 x1875771511468416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878193.789934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511468416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878193.789945:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878193.789950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878193.789954:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878193.789962:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878193.789989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878193.790003:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878193/real 1788878193] req@ffffa0618869ad80 x1875771511468544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878193.790019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511468544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878193.790028:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878193.790030:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878193.790033:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878193.790038:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878193.790056:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878193.790074:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878193/real 1788878193] req@ffffa06188698000 x1875771511468672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878193.790084:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511468672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878193.790090:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878193.790093:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878193.790095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878193.790100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878193.790113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878193.790120:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878193/real 1788878193] req@ffffa06188699f80 x1875771511468800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878193.790130:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511468800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878193.790136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878193.790138:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878193.790141:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511468800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878194.429537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878194.429541:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878194.685461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878195.453453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878195.453535:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878195.709475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878196.477480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878196.477533:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878196.733458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878197.501482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878197.501544:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878197.757466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878198.525473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878198.525541:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878198.781458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878198.909464:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878198.909479:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878198.909485:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878198.909511:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3185 00000100:00080000:1.0:1788878198.909526:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878198.909538:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878198.909587:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511468928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878198.909612:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878198.909616:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878198.909618:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878198.909622:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3185 00000100:00080000:1.0:1788878198.909625:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878198.909628:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878198.909640:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878198.909643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511468928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878198.909658:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511469056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878198.909673:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878198.909677:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878198.909679:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878198.909684:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3185 00000100:00080000:1.0:1788878198.909690:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878198.909693:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878198.909698:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878198.909709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511469184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878198.909717:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878198.909720:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878198.909722:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878198.909725:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3185 00000100:00080000:1.0:1788878198.909728:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878198.909730:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878198.909734:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878198.909750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878198.909756:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511469312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878198.909764:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878198.909767:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878198.909768:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878198.909770:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3185 00000100:00080000:1.0:1788878198.909781:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878198.909783:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878198.909787:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878198.909799:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878198/real 1788878198] req@ffffa06188699f80 x1875771511468928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878198.909801:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511469440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878198.909831:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511468928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878198.909841:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878198.909845:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878198.909850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511468928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878198.909874:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878198.909893:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878198.909901:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878198/real 1788878198] req@ffffa06188698000 x1875771511469056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878198.909910:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511469056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878198.909921:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878198.909924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878198.909926:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878198.909930:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878198.909941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878198.909947:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878198/real 1788878198] req@ffffa0618869ad80 x1875771511469184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878198.909954:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511469184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878198.909960:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878198.909961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878198.909963:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878198.909967:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878198.909978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878198.909984:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878198/real 1788878198] req@ffffa0618869bb80 x1875771511469312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878198.909992:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511469312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878198.909997:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878198.909999:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878198.910002:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878198.910006:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878198.910016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878198.910022:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878198/real 1788878198] req@ffffa0618869a680 x1875771511469440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878198.910029:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511469440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878198.910034:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878198.910036:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878198.910038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878199.549498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878199.549611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878199.805502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878200.573530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878200.573680:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878200.829462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878201.597477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878201.597533:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878201.853486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878202.621509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878202.621613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878202.877511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878203.645487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878203.645559:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878203.901485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878204.029491:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878204.029501:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878204.029505:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878204.029517:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3190 00000100:00080000:2.0:1788878204.029529:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878204.029537:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878204.029580:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511469568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878204.029605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878204.029608:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878204.029609:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878204.029613:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3190 00000100:00080000:2.0:1788878204.029615:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878204.029617:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878204.029627:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878204.029635:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511469696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878204.029642:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878204.029644:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878204.029645:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878204.029647:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3190 00000100:00080000:2.0:1788878204.029650:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878204.029652:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878204.029655:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878204.029665:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511469824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878204.029671:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878204.029674:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878204.029675:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878204.029678:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3190 00000100:00080000:2.0:1788878204.029680:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878204.029682:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878204.029685:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878204.029688:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511469568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878204.029695:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511469952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878204.029703:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878204.029704:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878204.029705:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878204.029707:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3190 00000100:00080000:2.0:1788878204.029709:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878204.029712:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878204.029714:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878204.029723:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511470080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878204.029777:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878204.029817:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878204/real 1788878204] req@ffffa061888aca80 x1875771511469568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878204.029834:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511469568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878204.029843:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878204.029846:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878204.029850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511469568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878204.029856:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878204.029872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878204.029882:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878204/real 1788878204] req@ffffa061888adc00 x1875771511469696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878204.029890:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511469696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878204.029896:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878204.029898:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878204.029901:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878204.029904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878204.029915:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878204.029920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878204/real 1788878204] req@ffffa061888ae300 x1875771511469824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878204.029928:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511469824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878204.029933:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878204.029935:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878204.029937:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878204.029968:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878204.029989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878204.030001:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878204/real 1788878204] req@ffffa061888af480 x1875771511469952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878204.030010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511469952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878204.030017:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878204.030019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878204.030022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511469952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878204.030025:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878204.030039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878204.030045:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878204/real 1788878204] req@ffffa061888ac000 x1875771511470080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878204.030052:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511470080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878204.030058:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878204.030060:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878204.030062:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878204.669496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878204.669549:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878204.925488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878205.693502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878205.693590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878205.949494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878206.717473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878206.717539:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878206.973519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878207.741483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878207.741591:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878207.997477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878208.765447:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878208.765558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878209.021513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878209.149480:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878209.149490:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878209.149495:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878209.149506:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3195 00000100:00080000:1.0:1788878209.149518:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878209.149526:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878209.149567:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511470208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878209.149591:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878209.149595:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878209.149596:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878209.149599:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3195 00000100:00080000:1.0:1788878209.149602:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878209.149604:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878209.149615:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878209.149632:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511470336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878209.149644:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878209.149648:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878209.149651:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878209.149655:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511470208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878209.149657:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3195 00000100:00080000:1.0:1788878209.149665:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878209.149672:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878209.149678:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878209.149693:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511470464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878209.149703:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878209.149706:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878209.149708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878209.149711:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3195 00000100:00080000:1.0:1788878209.149715:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878209.149717:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878209.149722:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878209.149730:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511470592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878209.149736:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878209.149738:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878209.149740:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878209.149742:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3195 00000100:00080000:1.0:1788878209.149743:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878209.149747:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788878209.149753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788878209.149754:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878209.149764:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511470720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878209.149797:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878209/real 1788878209] req@ffffa06188699c00 x1875771511470208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878209.149814:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511470208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878209.149823:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878209.149826:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878209.149830:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511470208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878209.149853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878209.149879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878209.149894:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878209/real 1788878209] req@ffffa0618869aa00 x1875771511470336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878209.149908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511470336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878209.149918:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878209.149922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878209.149926:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878209.149933:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878209.149954:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878209.149963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878209/real 1788878209] req@ffffa0618869b100 x1875771511470464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878209.149972:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511470464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878209.149977:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878209.149980:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878209.149982:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878209.149986:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878209.149998:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878209.150005:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878209/real 1788878209] req@ffffa06188698380 x1875771511470592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878209.150019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511470592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878209.150028:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878209.150031:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878209.150035:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878209.150041:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878209.150063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878209.150075:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878209/real 1788878209] req@ffffa0618869b800 x1875771511470720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878209.150087:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511470720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878209.150108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878209.150110:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878209.150113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878209.789526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878209.789640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878210.045504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878210.813490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878210.813583:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878211.069503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878211.837465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878211.837570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878212.093481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878212.861502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878212.861560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878213.117501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878213.885451:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878213.885581:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878214.141486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878214.269460:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878214.269469:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878214.269472:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878214.269481:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3200 00000100:00080000:1.0:1788878214.269490:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878214.269497:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878214.269541:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511470848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878214.269570:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878214.269574:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878214.269577:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878214.269582:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3200 00000100:00080000:1.0:1788878214.269585:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878214.269591:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878214.269601:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878214.269614:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511470976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878214.269618:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511470848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878214.269627:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878214.269631:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878214.269639:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878214.269643:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3200 00000100:00080000:1.0:1788878214.269646:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878214.269649:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878214.269653:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878214.269662:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511471104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878214.269670:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878214.269672:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878214.269674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878214.269676:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3200 00000100:00080000:1.0:1788878214.269679:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878214.269682:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878214.269685:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878214.269689:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878214.269701:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511471232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878214.269709:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878214.269712:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878214.269713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878214.269715:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3200 00000100:00080000:1.0:1788878214.269718:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878214.269721:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878214.269725:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878214.269729:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878214/real 1788878214] req@ffffa0618869b800 x1875771511470848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878214.269735:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511471360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878214.269758:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511470848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878214.269772:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878214.269775:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878214.269778:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511470848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878214.269795:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878214.269813:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878214.269822:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878214/real 1788878214] req@ffffa06188698380 x1875771511470976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878214.269830:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511470976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878214.269835:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878214.269838:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878214.269840:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511470976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878214.269843:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878214.269854:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878214.269859:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878214/real 1788878214] req@ffffa0618869b100 x1875771511471104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878214.269866:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511471104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878214.269872:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878214.269873:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878214.269875:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878214.269880:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878214.269890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878214.269896:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878214/real 1788878214] req@ffffa0618869aa00 x1875771511471232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878214.269903:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511471232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878214.269917:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878214.269918:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878214.269920:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878214.269924:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878214.269936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878214.269942:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878214/real 1788878214] req@ffffa06188699c00 x1875771511471360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878214.269949:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511471360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878214.269954:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878214.269956:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878214.269957:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878214.909522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878214.909585:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878215.165500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878215.933474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878215.933536:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878216.189448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878216.957485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878216.957531:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878217.213451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878217.981478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878217.981539:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878218.237449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878219.005477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878219.005536:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878219.261462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878219.389510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878219.389522:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878219.389529:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878219.389541:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3206 00000100:00080000:1.0:1788878219.389554:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878219.389562:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878219.389603:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511471488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878219.389632:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878219.389635:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878219.389636:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878219.389640:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3206 00000100:00080000:1.0:1788878219.389642:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878219.389645:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878219.389655:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878219.389663:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511471616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878219.389671:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878219.389675:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878219.389677:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878219.389681:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3206 00000100:00080000:1.0:1788878219.389685:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878219.389689:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878219.389694:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878219.389707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511471744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878219.389719:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878219.389723:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878219.389725:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878219.389731:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3206 00000100:00100000:3.0:1788878219.389733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511471488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878219.389734:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878219.389741:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878219.389746:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878219.389757:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511471872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878219.389764:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878219.389767:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878219.389769:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878219.389772:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3206 00000100:00080000:1.0:1788878219.389774:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878219.389777:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878219.389779:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878219.389788:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511472000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878219.389811:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878219.389850:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878219/real 1788878219] req@ffffa0618869a680 x1875771511471488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878219.389870:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511471488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878219.389882:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878219.389886:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878219.389891:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511471488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878219.389900:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878219.389925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878219.389940:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878219/real 1788878219] req@ffffa0618869bb80 x1875771511471616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878219.389954:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511471616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878219.389964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878219.389968:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878219.389971:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878219.389975:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878219.389992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878219.390000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878219/real 1788878219] req@ffffa0618869ad80 x1875771511471744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878219.390007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511471744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878219.390027:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878219.390029:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878219.390032:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878219.390078:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878219.390096:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878219.390104:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878219/real 1788878219] req@ffffa06188698000 x1875771511471872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878219.390112:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511471872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878219.390118:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878219.390120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878219.390122:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511471872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878219.390125:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878219.390135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878219.390141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878219/real 1788878219] req@ffffa06188699f80 x1875771511472000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878219.390149:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511472000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878219.390154:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878219.390155:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878219.390158:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878220.029511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878220.029624:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878220.285458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878221.053488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878221.053616:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878221.309464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878222.077509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878222.077599:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878222.333522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878223.101554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878223.101664:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878223.357443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878224.125501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878224.125560:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878224.381485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878224.509447:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878224.509456:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878224.509461:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878224.509471:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3211 00000100:00080000:1.0:1788878224.509480:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878224.509487:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878224.509521:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511472128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878224.509549:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878224.509553:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878224.509555:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878224.509560:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3211 00000100:00080000:1.0:1788878224.509564:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878224.509569:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878224.509579:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878224.509594:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511472256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878224.509606:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878224.509609:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878224.509612:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878224.509615:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3211 00000100:00080000:1.0:1788878224.509620:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878224.509625:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878224.509630:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511472128:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878224.509633:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878224.509648:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511472384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878224.509667:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878224.509669:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878224.509670:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878224.509674:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3211 00000100:00080000:1.0:1788878224.509677:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878224.509680:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878224.509684:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878224.509689:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878224.509700:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511472512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878224.509711:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878224.509714:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878224.509717:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878224.509721:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878224/real 1788878224] req@ffffa06188699f80 x1875771511472128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878224.509724:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3211 00000100:00080000:1.0:1788878224.509731:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878224.509738:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878224.509742:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511472128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878224.509745:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788878224.509757:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878224.509762:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788878224.509763:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511472640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878224.509767:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511472128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878224.509780:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878224.509797:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878224.509806:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878224/real 1788878224] req@ffffa06188698000 x1875771511472256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878224.509813:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511472256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878224.509819:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878224.509821:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878224.509823:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878224.509841:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878224.509851:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878224.509858:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878224/real 1788878224] req@ffffa0618869ad80 x1875771511472384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878224.509865:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511472384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878224.509870:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878224.509872:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878224.509874:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878224.509878:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878224.509887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878224.509892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878224/real 1788878224] req@ffffa0618869bb80 x1875771511472512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878224.509899:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511472512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878224.509904:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878224.509906:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878224.509908:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878224.509911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878224.509920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878224.509925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878224/real 1788878224] req@ffffa0618869a680 x1875771511472640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878224.509932:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511472640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878224.509937:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878224.509940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878224.509942:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878225.149516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878225.149627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878225.405517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878226.173476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878226.173558:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878226.429492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878227.197501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878227.197558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878227.453455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878228.221461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878228.221517:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878228.477497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878229.245480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878229.245542:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878229.501442:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878229.629504:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878229.629512:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878229.629517:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878229.629526:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3216 00000100:00080000:2.0:1788878229.629535:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878229.629541:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878229.629589:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511472768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878229.629620:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878229.629624:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878229.629625:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878229.629629:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3216 00000100:00080000:2.0:1788878229.629632:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878229.629637:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878229.629646:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878229.629658:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511472896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878229.629667:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878229.629670:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878229.629671:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878229.629691:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3216 00000100:00080000:2.0:1788878229.629699:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878229.629702:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511472768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878229.629709:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878229.629723:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878229.629744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511473024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878229.629758:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878229.629763:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878229.629766:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878229.629772:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3216 00000100:00080000:2.0:1788878229.629789:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878229.629796:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788878229.629806:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878229.629814:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878229.629839:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511473152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878229.629854:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878229/real 1788878229] req@ffffa061888adf80 x1875771511472768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878229.629856:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878229.629874:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878229.629879:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878229.629886:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511472768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878229.629887:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3216 00000100:00080000:2.0:1788878229.629917:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878229.629917:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878229.629924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788878229.629929:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878229.629931:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511472768:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878229.629947:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878229.629963:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511473280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878229.629968:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878229.630027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878229.630040:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878229/real 1788878229] req@ffffa061888ad500 x1875771511472896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878229.630049:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511472896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878229.630056:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878229.630059:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878229.630061:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511472896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878229.630066:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878229.630079:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878229.630085:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878229/real 1788878229] req@ffffa061888ac000 x1875771511473024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878229.630093:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511473024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878229.630104:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878229.630106:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878229.630108:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878229.630112:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878229.630125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878229.630132:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878229/real 1788878229] req@ffffa061888af480 x1875771511473152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878229.630139:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511473152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878229.630144:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878229.630146:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878229.630148:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878229.630160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878229.630171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878229.630177:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878229/real 1788878229] req@ffffa061888ae300 x1875771511473280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878229.630185:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511473280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878229.630191:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878229.630192:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878229.630194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878230.269548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878230.269625:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878230.525507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878231.293479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878231.293658:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878231.549529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878232.317556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878232.317654:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878232.573555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878233.341477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878233.341594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878233.597483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878234.365494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878234.365550:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878234.621525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878234.749488:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878234.749498:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878234.749502:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878234.749512:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3221 00000100:00080000:2.0:1788878234.749524:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878234.749532:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878234.749572:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511473408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878234.749605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878234.749609:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878234.749612:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878234.749617:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3221 00000100:00080000:2.0:1788878234.749649:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878234.749655:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878234.749663:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511473408:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878234.749672:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878234.749688:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511473536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878234.749696:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878234.749716:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878234.749718:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878234.749721:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3221 00000100:00080000:2.0:1788878234.749727:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878234.749731:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878234.749735:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788878234.749742:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878234.749747:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511473664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878234.749756:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878234.749759:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878234.749760:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878234.749763:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3221 00000100:00080000:2.0:1788878234.749767:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878234.749771:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878234.749775:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878234.749780:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878234/real 1788878234] req@ffffa061888adc00 x1875771511473408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878234.749787:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511473792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878234.749804:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511473408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878234.749805:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878234.749832:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878234.749835:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878234.749840:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3221 00000100:00080000:2.0:1788878234.749848:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878234.749851:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878234.749855:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788878234.749859:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:2.0:1788878234.749863:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878234.749864:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511473408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878234.749879:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511473920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878234.749909:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878234.749935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878234.749946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878234/real 1788878234] req@ffffa061888aca80 x1875771511473536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878234.749972:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511473536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878234.750004:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878234.750007:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878234.750010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878234.750031:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878234.750047:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878234.750055:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878234/real 1788878234] req@ffffa061888afb80 x1875771511473664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878234.750063:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511473664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878234.750069:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878234.750071:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878234.750074:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878234.750077:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878234.750105:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878234.750112:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878234/real 1788878234] req@ffffa061888af100 x1875771511473792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878234.750129:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511473792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878234.750135:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878234.750137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878234.750140:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878234.750170:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878234.750183:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878234.750194:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878234/real 1788878234] req@ffffa061888ae680 x1875771511473920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878234.750207:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511473920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878234.750216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878234.750219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878234.750223:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511473920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878235.389514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878235.389546:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878235.645521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878236.413478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878236.413533:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878236.669474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878237.437495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878237.437559:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878237.693501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878238.461505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878238.461510:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878238.717561:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878239.485514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878239.485657:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878239.741574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878239.869540:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878239.869552:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878239.869557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878239.869568:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3226 00000100:00080000:1.0:1788878239.869581:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878239.869590:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878239.869633:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511474048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878239.869665:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878239.869670:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878239.869672:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878239.869677:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3226 00000100:00080000:1.0:1788878239.869680:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878239.869686:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878239.869696:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878239.869709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511474176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878239.869724:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878239.869730:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878239.869733:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878239.869740:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3226 00000100:00080000:1.0:1788878239.869747:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878239.869754:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878239.869763:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878239.869780:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511474048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878239.869787:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511474304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878239.869810:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878239.869816:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878239.869819:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878239.869824:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3226 00000100:00080000:1.0:1788878239.869829:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878239.869835:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878239.869842:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878239.869859:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511474432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878239.869870:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878239.869875:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878239.869877:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878239.869880:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3226 00000100:00080000:1.0:1788878239.869885:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878239.869888:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878239.869893:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878239.869904:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511474560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878239.869935:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878239.870023:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878239.870076:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878239.870105:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878240.509463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878240.509527:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878240.765455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878241.533483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878241.533548:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878241.789534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878242.557502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878242.557586:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878242.813506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878242.941830:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878242.941891:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878242.941909:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878242.941990:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878242.942005:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878242.942015:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878242.942027:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788878242.942076:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878239/real 1788878242] req@ffffa0618869b100 x1875771511474304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878242.942126:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511474304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878242.942146:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878242.942151:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878242.942155:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878242.942163:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878239/real 1788878242] req@ffffa06188698380 x1875771511474432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878242.942176:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511474432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878242.942184:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878242.942187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878242.942190:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878242.942195:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878239/real 1788878242] req@ffffa0618869b800 x1875771511474560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878242.942205:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511474560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878242.942214:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878242.942220:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878242.942225:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878242.942241:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878239/real 1788878242] req@ffffa06188699c00 x1875771511474048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878242.942259:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511474048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878242.942270:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878242.942275:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878242.942280:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511474048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878242.942290:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878239/real 1788878242] req@ffffa0618869aa00 x1875771511474176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878242.942306:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511474176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878242.942317:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878242.942320:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878242.942325:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878243.581556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878243.581660:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878243.837546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878244.605460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878244.605568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878244.861472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878244.989621:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878244.989633:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878244.989640:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878244.989653:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3231 00000100:00080000:2.0:1788878244.989667:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878244.989678:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878244.989722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511474688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878244.989752:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878244.989757:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878244.989759:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878244.989764:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3231 00000100:00080000:2.0:1788878244.989777:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878244.989783:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878244.989795:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878244.989811:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511474816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878244.989826:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878244.989830:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878244.989836:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878244.989839:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511474688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878244.989844:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3231 00000100:00080000:2.0:1788878244.989853:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878244.989861:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878244.989869:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878244.989891:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511474944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878244.989921:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878244.989928:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878244.989960:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878244.989968:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3231 00000100:00080000:2.0:1788878244.989977:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878244.989985:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878244.989995:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878244.989997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878244.990029:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511475072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878244.990051:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878244.990057:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878244.990066:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878244.990077:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3231 00000100:00080000:2.0:1788878244.990085:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878244.990092:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878244.990094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878244/real 1788878244] req@ffffa061888ac380 x1875771511474688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878244.990103:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878244.990139:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511475200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878244.990150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511474688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878244.990184:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878244.990194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878244.990224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511474688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878244.990267:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878244.990332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878244.990360:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878244/real 1788878244] req@ffffa061888aea00 x1875771511474816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878244.990433:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511474816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878244.990452:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878244.990459:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878244.990465:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878244.990478:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878244.990521:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878244.990546:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878244/real 1788878244] req@ffffa061888ad880 x1875771511474944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878244.990570:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511474944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878244.990586:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878244.990592:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878244.990598:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511474944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878244.990610:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878244.991727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878244.991768:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878244/real 1788878244] req@ffffa061888aed80 x1875771511475072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878244.991809:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511475072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878244.991825:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878244.991831:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878244.991838:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878244.991852:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878244.991907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878244.991962:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878244/real 1788878244] req@ffffa061888ac700 x1875771511475200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878244.992001:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511475200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878244.992019:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878244.992026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878244.992032:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878245.629572:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878245.629701:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878245.885503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878246.653486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878246.653620:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878246.909481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878247.677514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878247.677624:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878247.933532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878248.701457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878248.701508:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878248.957532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878249.725515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878249.725590:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878249.981533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878250.109510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878250.109522:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878250.109528:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878250.109543:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3236 00000100:00080000:1.0:1788878250.109590:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878250.109604:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878250.109660:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511475328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878250.109696:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878250.109702:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878250.109704:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878250.109710:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3236 00000100:00080000:1.0:1788878250.109714:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878250.109718:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878250.109731:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878250.109747:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511475328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878250.109756:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511475456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878250.109769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878250.109773:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878250.109775:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878250.109780:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3236 00000100:00080000:1.0:1788878250.109784:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878250.109787:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878250.109794:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878250.109824:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511475584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878250.109833:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878250.109836:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878250.109838:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878250.109842:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3236 00000100:00080000:1.0:1788878250.109846:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878250.109849:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878250.109854:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878250.109863:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511475712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878250.109870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878250.109882:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878250.109886:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878250.109888:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878250.109893:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3236 00000100:00080000:1.0:1788878250.109897:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878250.109901:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878250.109905:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878250.109943:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878250/real 1788878250] req@ffffa0618869aa00 x1875771511475328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878250.109944:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511475840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878250.110001:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511475328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878250.110015:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878250.110019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878250.110024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511475328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878250.110050:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878250.110073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878250.110085:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878250/real 1788878250] req@ffffa06188699c00 x1875771511475456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878250.110096:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511475456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878250.110104:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878250.110107:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878250.110110:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878250.110115:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878250.110129:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878250.110137:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878250/real 1788878250] req@ffffa0618869b800 x1875771511475584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878250.110153:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511475584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878250.110160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878250.110165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878250.110168:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878250.110174:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878250.110191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878250.110201:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878250/real 1788878250] req@ffffa06188698380 x1875771511475712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878250.110212:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511475712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878250.110219:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878250.110221:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878250.110224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878250.110229:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878250.110242:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878250.110250:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878250/real 1788878250] req@ffffa0618869b100 x1875771511475840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878250.110261:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511475840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878250.110277:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878250.110280:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878250.110284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511475840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878250.749457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878250.749550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878251.005491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878251.773471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878251.773595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878252.029523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878252.797509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878252.797738:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878253.053484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878253.821463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878253.821547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878254.077483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878254.845454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878254.845576:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878255.101502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878255.229553:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878255.229563:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878255.229568:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878255.229577:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3241 00000100:00080000:2.0:1788878255.229589:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878255.229596:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878255.229631:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511475968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878255.229654:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878255.229658:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878255.229660:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878255.229663:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3241 00000100:00080000:2.0:1788878255.229667:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878255.229671:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878255.229680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878255.229689:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511475968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878255.229696:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511476096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878255.229713:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878255.229717:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878255.229720:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878255.229725:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3241 00000100:00080000:2.0:1788878255.229731:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878255.229737:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878255.229745:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878255.229762:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878255.229765:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511476224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878255.229796:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878255.229800:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878255.229803:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878255.229808:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3241 00000100:00080000:2.0:1788878255.229814:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878255.229819:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878255.229826:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878255.229827:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878255/real 1788878255] req@ffffa061888ace00 x1875771511475968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878255.229844:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511476352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878255.229862:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878255.229866:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878255.229868:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878255.229875:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511475968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878255.229876:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3241 00000100:00080000:2.0:1788878255.229917:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878255.229926:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878255.229934:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878255.229936:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788878255.229947:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511476480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878255.229952:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878255.229962:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511475968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878255.229991:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878255.230014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878255.230026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878255/real 1788878255] req@ffffa061888ad180 x1875771511476096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878255.230057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511476096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878255.230076:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878255.230080:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878255.230084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878255.230092:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878255.230113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878255.230141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878255/real 1788878255] req@ffffa061888ac700 x1875771511476224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878255.230150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511476224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878255.230156:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878255.230158:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878255.230161:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878255.230166:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878255.230188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878255.230200:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878255/real 1788878255] req@ffffa061888aed80 x1875771511476352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878255.230215:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511476352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878255.230225:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878255.230228:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878255.230232:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878255.230259:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878255.230289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878255.230342:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878255/real 1788878255] req@ffffa061888ad880 x1875771511476480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878255.230365:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511476480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878255.230393:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878255.230426:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878255.230432:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878255.869518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878255.869665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878256.125524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878256.893497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878256.893566:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878257.149457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878257.917449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878257.917529:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878258.173568:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878258.941475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878258.941532:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878259.197489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878259.965578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878259.965657:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878260.221522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878260.349600:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878260.349611:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878260.349616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878260.349650:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3247 00000100:00080000:1.0:1788878260.349663:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878260.349673:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878260.349735:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511476608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878260.349779:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878260.349784:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878260.349787:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878260.349792:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3247 00000100:00080000:1.0:1788878260.349798:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878260.349803:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878260.349815:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878260.349841:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511476736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878260.349842:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511476608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878260.349911:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878260.349916:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878260.349919:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878260.349923:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3247 00000100:00080000:1.0:1788878260.349929:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878260.349934:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878260.349941:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878260.349957:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511476864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878260.349969:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878260.349979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878260.349980:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878260.349988:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878260.349994:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3247 00000100:00080000:1.0:1788878260.350001:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878260.350006:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878260.350014:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878260.350030:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511476992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878260.350042:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878260/real 1788878260] req@ffffa0618869a680 x1875771511476608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878260.350048:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878260.350058:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878260.350063:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878260.350075:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3247 00000100:00080000:1.0:1788878260.350082:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878260.350093:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878260.350094:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511476608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878260.350098:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878260.350118:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788878260.350122:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511477120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878260.350128:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878260.350144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511476608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878260.350183:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878260.350210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878260.350224:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878260/real 1788878260] req@ffffa0618869bb80 x1875771511476736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878260.350239:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511476736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878260.350249:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878260.350252:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878260.350256:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878260.350263:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878260.350299:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878260.350311:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878260/real 1788878260] req@ffffa0618869ad80 x1875771511476864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878260.350325:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511476864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878260.350335:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878260.350339:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878260.350350:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878260.350357:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878260.350378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878260.350434:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878260/real 1788878260] req@ffffa06188698000 x1875771511476992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878260.350451:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511476992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878260.350485:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878260.350488:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878260.350492:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511476992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878260.350499:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878260.350525:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878260.350539:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878260/real 1788878260] req@ffffa06188699f80 x1875771511477120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878260.350554:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511477120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878260.350569:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878260.350575:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878260.350581:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878260.989480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878260.989613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878261.245501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878262.013503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878262.013520:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878262.269572:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878263.037489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878263.037657:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878263.293502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878264.061506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878264.061611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878264.317509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878265.085543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878265.085625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878265.341482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878265.469514:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878265.469522:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878265.469526:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878265.469534:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3252 00000100:00080000:3.0:1788878265.469543:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878265.469550:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878265.469602:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511477248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878265.469633:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878265.469637:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878265.469639:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878265.469644:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3252 00000100:00080000:3.0:1788878265.469649:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878265.469654:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878265.469665:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878265.469698:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511477248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878265.469701:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511477376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878265.469715:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878265.469719:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878265.469721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878265.469725:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3252 00000100:00080000:3.0:1788878265.469730:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878265.469735:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878265.469741:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878265.469754:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511477504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878265.469759:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878265.469769:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878265.469772:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878265.469773:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878265.469777:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3252 00000100:00080000:3.0:1788878265.469781:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878265.469785:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878265.469789:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878265.469794:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878265/real 1788878265] req@ffffa06188871c00 x1875771511477248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878265.469799:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511477632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878265.469823:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878265.469831:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878265.469832:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878265.469836:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3252 00000100:00080000:3.0:1788878265.469849:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878265.469851:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511477248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878265.469858:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878265.469864:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878265.469865:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878265.469872:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878265.469875:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511477248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878265.469879:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511477760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878265.469900:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878265.469919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878265.469937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878265/real 1788878265] req@ffffa06188870380 x1875771511477376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878265.469946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511477376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878265.469952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878265.469954:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878265.469957:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878265.469961:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878265.469974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878265.469981:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878265/real 1788878265] req@ffffa06188870a80 x1875771511477504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878265.469988:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511477504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878265.470004:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878265.470008:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878265.470010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878265.470017:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878265.470040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878265.470050:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878265/real 1788878265] req@ffffa06188871880 x1875771511477632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878265.470058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511477632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878265.470063:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878265.470065:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878265.470067:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878265.470071:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878265.470083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878265.470089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878265/real 1788878265] req@ffffa06188872d80 x1875771511477760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878265.470096:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511477760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878265.470101:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878265.470102:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878265.470105:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511477760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878266.109557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878266.109626:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878266.365507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878267.133513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878267.133595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878267.389518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878268.157484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878268.157719:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878268.413546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878269.181456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878269.181511:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878269.437518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878270.205528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878270.205604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878270.461546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878270.589563:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878270.589576:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878270.589582:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878270.589597:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3257 00000100:00080000:1.0:1788878270.589611:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878270.589620:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878270.589671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511477888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878270.589707:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878270.589713:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878270.589716:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878270.589721:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3257 00000100:00080000:1.0:1788878270.589736:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878270.589742:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878270.589755:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878270.589774:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511477888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878270.589775:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511478016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878270.589802:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878270.589807:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878270.589809:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878270.589841:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3257 00000100:00080000:1.0:1788878270.589848:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878270.589853:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878270.589862:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878270.589883:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511478144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878270.589903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878270.589908:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878270.589918:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878270.589922:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878270.589933:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3257 00000100:00080000:1.0:1788878270.589943:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878270.589952:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878270.589960:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878270/real 1788878270] req@ffffa06188699f80 x1875771511477888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878270.589967:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878270.590000:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511478272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878270.590001:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511477888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878270.590058:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878270.590061:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878270.590069:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878270.590076:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788878270.590078:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878270.590092:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3257 00000100:00100000:0.0:1788878270.590094:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511477888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878270.590098:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878270.590111:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878270.590119:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878270.590135:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511478400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878270.590155:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878270.590208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878270.590231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878270/real 1788878270] req@ffffa06188698000 x1875771511478016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878270.590248:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511478016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878270.590260:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878270.590264:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878270.590268:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878270.590277:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878270.590301:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878270.590313:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878270/real 1788878270] req@ffffa0618869ad80 x1875771511478144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878270.590328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511478144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878270.590337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878270.590341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878270.590344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878270.590352:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878270.590371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878270.590416:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878270/real 1788878270] req@ffffa0618869bb80 x1875771511478272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878270.590432:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511478272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878270.590446:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878270.590450:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878270.590454:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878270.590476:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878270.590500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878270.590514:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878270/real 1788878270] req@ffffa0618869a680 x1875771511478400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878270.590528:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511478400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878270.590548:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878270.590551:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878270.590555:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878271.229503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878271.229584:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878271.485525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878272.253518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878272.253623:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878272.509535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878273.277555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878273.277638:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878273.533505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878274.301492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878274.301630:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878274.557543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878275.325529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878275.325628:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878275.581526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878275.709659:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878275.709673:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878275.709679:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878275.709691:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3262 00000100:00080000:3.0:1788878275.709705:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878275.709716:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878275.709829:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511478528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878275.709883:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878275.709891:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878275.709893:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878275.709898:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3262 00000100:00080000:3.0:1788878275.709903:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878275.709908:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878275.709919:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878275.709938:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511478528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878275.709938:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511478656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878275.709969:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878275.709972:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878275.709974:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878275.709979:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3262 00000100:00080000:3.0:1788878275.709982:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878275.709985:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878275.709990:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878275.710002:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511478784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878275.710011:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878275.710015:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878275.710017:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878275.710020:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3262 00000100:00080000:3.0:1788878275.710024:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878275.710030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878275.710033:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878275.710041:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878275.710053:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511478912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878275.710064:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878275.710067:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878275.710069:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878275.710083:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3262 00000100:00080000:3.0:1788878275.710087:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878275.710093:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878275.710095:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878275/real 1788878275] req@ffffa06188872680 x1875771511478528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878275.710097:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878275.710112:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511479040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878275.710144:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511478528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878275.710156:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878275.710160:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878275.710164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511478528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878275.710189:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878275.710213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878275.710224:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878275/real 1788878275] req@ffffa06188870700 x1875771511478656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878275.710242:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511478656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878275.710250:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878275.710253:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878275.710257:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878275.710265:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878275.710321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878275.710341:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878275/real 1788878275] req@ffffa06188873b80 x1875771511478784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878275.710363:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511478784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878275.710380:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878275.710384:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878275.710388:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878275.710423:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878275.710456:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878275.710476:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878275/real 1788878275] req@ffffa06188872300 x1875771511478912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878275.710499:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511478912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878275.710525:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878275.710529:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878275.710533:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511478912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878275.710540:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878275.710573:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878275.710586:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878275/real 1788878275] req@ffffa06188873480 x1875771511479040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878275.710599:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511479040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878275.710607:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878275.710611:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878275.710614:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878276.349497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878276.349633:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878276.605592:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878277.373490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878277.373649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878277.629480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878278.397514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878278.397558:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878278.653469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878279.421484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878279.421592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878279.677472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878280.445472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878280.445547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878280.701464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878280.829593:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878280.829609:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878280.829616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878280.829643:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3267 00000100:00080000:1.0:1788878280.829660:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878280.829672:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878280.829744:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511479168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878280.829822:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878280.829827:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878280.829829:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878280.829835:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3267 00000100:00080000:1.0:1788878280.829840:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878280.829846:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878280.829862:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878280.829886:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511479296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878280.829889:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511479168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878280.829930:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878280.829936:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878280.829940:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878280.829948:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3267 00000100:00080000:1.0:1788878280.829954:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878280.829960:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878280.829968:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878280.829983:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511479424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878280.830003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878280.830003:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878280.830015:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878280.830019:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878280.830028:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3267 00000100:00080000:1.0:1788878280.830037:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878280.830045:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878280.830054:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878280.830068:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878280/real 1788878280] req@ffffa0618869b100 x1875771511479168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878280.830075:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511479552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878280.830151:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878280.830161:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511479168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878280.830162:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878280.830173:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878280.830198:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3267 00000100:00080000:1.0:1788878280.830214:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878280.830236:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878280.830242:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788878280.830245:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878280.830257:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878280.830264:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511479168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878280.830273:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511479680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878280.830298:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878280.830341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878280.830362:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878280/real 1788878280] req@ffffa06188698380 x1875771511479296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878280.830415:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511479296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878280.830426:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878280.830429:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878280.830433:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878280.830440:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878280.830464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878280.830476:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878280/real 1788878280] req@ffffa0618869b800 x1875771511479424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878280.830486:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511479424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878280.830493:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878280.830496:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878280.830499:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878280.830506:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878280.830522:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878280.830531:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878280/real 1788878280] req@ffffa06188699c00 x1875771511479552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878280.830542:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511479552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878280.830549:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878280.830551:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878280.830554:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878280.830559:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878280.830574:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878280.830582:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878280/real 1788878280] req@ffffa0618869aa00 x1875771511479680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878280.830599:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511479680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878280.830610:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878280.830615:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878280.830620:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878281.469532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878281.469605:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878281.725513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878282.493471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878282.493581:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878282.749491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878283.517492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878283.517638:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878283.773467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878284.541492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878284.541597:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878284.797487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878285.565468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878285.565531:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878285.821467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878285.949540:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878285.949556:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878285.949563:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878285.949590:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3272 00000100:00080000:3.0:1788878285.949605:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878285.949616:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878285.949666:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511479808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878285.949702:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878285.949708:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878285.949711:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878285.949716:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3272 00000100:00080000:3.0:1788878285.949722:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878285.949730:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878285.949793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878285.949811:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511479808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878285.949859:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511479936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878285.949887:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878285.949895:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878285.949898:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878285.949907:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3272 00000100:00080000:3.0:1788878285.949916:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878285.949921:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878285.949932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788878285.949933:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878285.949961:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511480064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878285.949977:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878285.950009:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878285.950013:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878285.950021:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878285/real 1788878285] req@ffffa06188873800 x1875771511479808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878285.950023:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3272 00000100:00080000:3.0:1788878285.950034:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878285.950059:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878285.950068:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878285.950075:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511479808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878285.950089:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511480192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878285.950092:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878285.950105:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878285.950116:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511479808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878285.950141:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878285.950146:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878285.950149:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878285.950157:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3272 00000100:00080000:3.0:1788878285.950162:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878285.950167:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878285.950173:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479936:192.168.203.111@tcp:38:kworker.0 00000008:00100000:3.0:1788878285.950177:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878285.950198:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511480320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878285.950214:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878285.950242:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878285/real 1788878285] req@ffffa06188872a00 x1875771511479936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878285.950258:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511479936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878285.950268:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878285.950271:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878285.950275:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511479936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878285.950281:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878285.950298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878285.950308:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878285/real 1788878285] req@ffffa06188873100 x1875771511480064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878285.950329:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511480064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878285.950337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878285.950340:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878285.950343:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878285.950348:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878285.950365:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878285.950374:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878285/real 1788878285] req@ffffa06188871500 x1875771511480192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878285.950386:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511480192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878285.950395:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878285.950397:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878285.950401:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878285.950415:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878285.950462:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878285.950474:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878285/real 1788878285] req@ffffa06188873800 x1875771511480320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878285.950487:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511480320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878285.950503:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878285.950506:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878285.950510:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878286.589463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878286.589525:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878286.845542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878287.613450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878287.613510:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878287.869478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878288.637499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878288.637582:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878288.893475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878289.661515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878289.661610:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878289.917490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878290.685547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878290.685684:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878290.941518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878291.069517:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878291.069534:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878291.069541:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878291.069557:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3277 00000100:00080000:3.0:1788878291.069575:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878291.069588:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878291.069643:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511480448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878291.069709:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878291.069712:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878291.069714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878291.069718:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3277 00000100:00080000:3.0:1788878291.069738:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878291.069743:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878291.069760:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878291.069767:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511480448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878291.069777:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511480576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878291.069788:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878291.069791:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878291.069793:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878291.069795:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3277 00000100:00080000:3.0:1788878291.069817:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878291.069822:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878291.069830:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878291.069846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511480704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878291.069859:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878291.069865:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788878291.069871:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878291.069873:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878291.069886:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3277 00000100:00080000:3.0:1788878291.069895:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878291.069903:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878291.069915:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878291.069923:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878291/real 1788878291] req@ffffa06188873480 x1875771511480448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878291.069957:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511480832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878291.069965:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511480448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878291.069988:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878291.069989:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878291.069995:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878291.069997:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788878291.070002:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511480448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878291.070004:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878291.070013:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3277 00000100:00080000:3.0:1788878291.070019:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878291.070027:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480576:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788878291.070035:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878291.070042:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878291.070056:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511480960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878291.070066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878291.070088:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878291/real 1788878291] req@ffffa06188872300 x1875771511480576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878291.070102:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511480576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878291.070115:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878291.070119:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878291.070124:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878291.070131:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878291.070156:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878291.070171:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878291/real 1788878291] req@ffffa06188873b80 x1875771511480704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878291.070185:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511480704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878291.070191:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878291.070194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878291.070196:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878291.070201:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878291.070218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878291.070226:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878291/real 1788878291] req@ffffa06188870700 x1875771511480832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878291.070234:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511480832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878291.070240:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878291.070242:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878291.070245:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878291.070259:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878291.070271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878291.070298:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878291/real 1788878291] req@ffffa06188872680 x1875771511480960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878291.070311:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511480960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878291.070321:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878291.070324:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878291.070327:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511480960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878291.709513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878291.709549:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878291.965512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878292.733507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878292.733590:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878292.989491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878293.757584:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878293.757627:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878294.013492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878294.781488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878294.781547:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878295.037538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878295.805545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878295.805814:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878296.061517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878296.189514:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878296.189527:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878296.189533:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878296.189550:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3282 00000100:00080000:3.0:1788878296.189570:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878296.189585:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878296.189643:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511481088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878296.189687:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878296.189691:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878296.189694:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878296.189720:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3282 00000100:00080000:3.0:1788878296.189729:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878296.189746:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878296.189760:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878296.189772:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511481088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878296.189788:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511481216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878296.189814:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878296.189819:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878296.189822:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878296.189826:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3282 00000100:00080000:3.0:1788878296.189831:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878296.189846:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878296.189852:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878296.189867:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511481344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878296.189878:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878296.189883:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878296.189886:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878296.189890:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3282 00000100:00080000:3.0:1788878296.189893:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878296.189897:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878296.189902:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878296.189916:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511481472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878296.189931:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878296.189943:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878296.189949:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878296.189952:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878296.189957:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3282 00000100:00080000:3.0:1788878296.189962:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878296.189966:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878296.189971:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878296.189984:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511481600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878296.190038:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878296/real 1788878296] req@ffffa06188872d80 x1875771511481088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878296.190061:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511481088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878296.190073:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878296.190078:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878296.190082:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511481088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878296.190136:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878296.190161:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878296.190172:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878296/real 1788878296] req@ffffa06188871880 x1875771511481216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878296.190181:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511481216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878296.190188:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878296.190191:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878296.190194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878296.190199:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878296.190212:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878296.190219:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878296/real 1788878296] req@ffffa06188870a80 x1875771511481344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878296.190228:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511481344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878296.190234:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878296.190236:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878296.190238:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878296.190245:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878296.190268:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878296.190282:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878296/real 1788878296] req@ffffa06188870380 x1875771511481472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878296.190298:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511481472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878296.190309:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878296.190312:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878296.190315:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878296.190320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878296.190338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878296.190347:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878296/real 1788878296] req@ffffa06188871c00 x1875771511481600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878296.190356:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511481600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878296.190363:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878296.190366:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878296.190368:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878296.829467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878296.829593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878297.085539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878297.853521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878297.853727:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878298.109589:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878298.877465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878298.877570:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878299.133596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878299.901492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878299.901557:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878300.157555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878300.925546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878300.925634:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878301.181488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878301.309506:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878301.309520:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878301.309525:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878301.309540:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3288 00000100:00080000:2.0:1788878301.309566:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878301.309579:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878301.309630:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511481728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878301.309661:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878301.309668:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878301.309672:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878301.309699:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3288 00000100:00080000:2.0:1788878301.309707:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878301.309718:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511481728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878301.309719:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878301.309740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878301.309771:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511481856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878301.309792:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878301.309798:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878301.309802:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878301.309808:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3288 00000800:00000100:0.0:1788878301.309816:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878301.309817:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878301.309827:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878301.309835:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878301.309852:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511481984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878301.309864:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878301.309868:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878301.309871:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878301.309875:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3288 00000100:00080000:2.0:1788878301.309880:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878301.309888:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878301.309889:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878301/real 1788878301] req@ffffa061888aea00 x1875771511481728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878301.309895:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878301.309914:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511482112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878301.309943:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878301.309947:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878301.309950:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878301.309957:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3288 00000100:00100000:0.0:1788878301.309959:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511481728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878301.309961:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878301.309970:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878301.309978:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878301.309994:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511482240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878301.309996:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878301.310013:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878301.310022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511481728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878301.310052:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878301.310082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878301.310098:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878301/real 1788878301] req@ffffa061888ac380 x1875771511481856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878301.310118:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511481856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878301.310132:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878301.310138:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878301.310144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878301.310154:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878301.310188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878301.310203:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878301/real 1788878301] req@ffffa061888ae680 x1875771511481984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878301.310215:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511481984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878301.310230:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878301.310233:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878301.310237:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511481984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878301.310243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878301.310261:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878301.310270:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878301/real 1788878301] req@ffffa061888af100 x1875771511482112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878301.310281:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511482112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878301.310289:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878301.310292:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878301.310296:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878301.310301:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878301.310318:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878301.310327:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878301/real 1788878301] req@ffffa061888afb80 x1875771511482240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878301.310337:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511482240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878301.310345:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878301.310347:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878301.310351:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878301.949494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878301.949565:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878302.205477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878302.973504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878302.973635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878303.229454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878303.997486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878303.997640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878304.253506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878305.021459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878305.021555:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878305.277497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878306.045487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878306.045568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878306.301464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878306.429561:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878306.429572:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878306.429576:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878306.429588:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3293 00000100:00080000:1.0:1788878306.429600:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878306.429608:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878306.429645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511482368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878306.429695:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878306.429699:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878306.429701:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878306.429706:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3293 00000100:00080000:1.0:1788878306.429711:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878306.429717:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878306.429728:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878306.429750:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511482496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878306.429755:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511482368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878306.429775:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878306.429779:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878306.429783:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878306.429791:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3293 00000100:00080000:1.0:1788878306.429796:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878306.429801:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878306.429808:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878306.429841:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511482624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878306.429853:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878306.429858:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878306.429861:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878306.429865:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3293 00000100:00080000:1.0:1788878306.429873:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878306.429878:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878306.429885:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878306.429904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788878306.429922:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511482752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878306.429936:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482624:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788878306.429940:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878306.429948:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878306.429951:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878306.429975:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3293 00000100:00100000:0.0:1788878306.429980:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482752:192.168.203.111@tcp:8:kworker.0 00000100:00080000:1.0:1788878306.429983:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878306.429991:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878306.430001:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878306.430019:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511482880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878306.430060:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878307.069472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878307.069605:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878307.325451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878308.093480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878308.093549:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878308.349499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878309.117462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878309.117525:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878309.373500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788878309.501789:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788878309.501829:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788878309.501855:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878309.501890:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878309.501898:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878309.501904:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878309.501912:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878309.501941:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878306/real 1788878309] req@ffffa0618869b100 x1875771511482880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878309.501984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511482880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878309.501999:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878309.502004:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878309.502010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878309.502020:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878306/real 1788878309] req@ffffa06188698380 x1875771511482752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878309.502046:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511482752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878309.502056:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878309.502060:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878309.502064:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878309.502073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878306/real 1788878309] req@ffffa06188699c00 x1875771511482496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878309.502097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511482496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878309.502107:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878309.502112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878309.502117:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878309.502125:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878306/real 1788878309] req@ffffa0618869b800 x1875771511482624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878309.502141:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511482624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878309.502163:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878309.502167:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878309.502171:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511482624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878309.502181:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878306/real 1788878309] req@ffffa0618869aa00 x1875771511482368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878309.502196:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511482368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878309.502224:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878309.502229:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878309.502235:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511482368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878310.141482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878310.141556:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878310.397544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878311.165465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878311.165509:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878311.421459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878311.549490:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878311.549499:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878311.549502:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878311.549510:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3298 00000100:00080000:1.0:1788878311.549519:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878311.549526:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878311.549562:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511483008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878311.549590:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878311.549594:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878311.549595:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878311.549599:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3298 00000100:00080000:1.0:1788878311.549601:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878311.549605:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878311.549613:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878311.549622:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511483136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878311.549644:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878311.549647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878311.549649:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878311.549652:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3298 00000100:00080000:1.0:1788878311.549655:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878311.549657:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878311.549662:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878311.549668:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511483008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878311.549673:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511483264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878311.549683:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878311.549685:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878311.549686:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878311.549689:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3298 00000100:00080000:1.0:1788878311.549692:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878311.549694:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878311.549697:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878311.549705:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511483392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878311.549718:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878311.549722:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878311.549724:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878311.549731:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3298 00000100:00080000:1.0:1788878311.549735:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788878311.549741:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878311.549745:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878311.549751:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878311.549767:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511483520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878311.549781:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878311/real 1788878311] req@ffffa0618869a680 x1875771511483008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878311.549802:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511483008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878311.549820:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878311.549824:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878311.549828:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511483008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878311.549835:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878311.549859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878311.549872:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878311/real 1788878311] req@ffffa0618869bb80 x1875771511483136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878311.549885:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511483136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878311.549894:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878311.549898:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878311.549902:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878311.549929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878311.549952:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878311.549963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878311/real 1788878311] req@ffffa0618869ad80 x1875771511483264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878311.549974:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511483264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878311.549982:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878311.549985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878311.549988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878311.549995:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878311.550029:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878311.550041:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878311/real 1788878311] req@ffffa06188698000 x1875771511483392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878311.550055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511483392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878311.550063:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878311.550066:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878311.550069:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878311.550075:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878311.550094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878311.550105:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878311/real 1788878311] req@ffffa06188699f80 x1875771511483520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878311.550117:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511483520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878311.550126:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878311.550129:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878311.550132:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878312.189458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878312.189541:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878312.445566:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878313.213500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878313.213594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878313.469455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878314.237460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878314.237588:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878314.493479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878315.261525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878315.261590:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878315.517483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878316.285466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878316.285539:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878316.541438:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878316.669474:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878316.669482:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878316.669486:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878316.669496:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3303 00000100:00080000:1.0:1788878316.669505:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878316.669523:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878316.669562:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511483648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878316.669590:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878316.669593:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878316.669594:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878316.669598:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3303 00000100:00080000:1.0:1788878316.669605:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878316.669619:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878316.669627:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878316.669644:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511483776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878316.669653:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878316.669656:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511483648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878316.669657:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878316.669660:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878316.669668:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3303 00000100:00080000:1.0:1788878316.669671:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878316.669673:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878316.669678:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878316.669687:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511483904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878316.669694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878316.669696:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878316.669698:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878316.669700:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3303 00000100:00080000:1.0:1788878316.669702:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878316.669705:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878316.669708:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878316.669716:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511484032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878316.669720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878316.669728:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878316.669731:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878316.669733:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878316.669736:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3303 00000100:00080000:1.0:1788878316.669740:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878316.669743:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878316.669747:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878316.669760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511484160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878316.669760:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878316/real 1788878316] req@ffffa06188699f80 x1875771511483648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878316.669801:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511483648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878316.669809:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878316.669813:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878316.669815:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511483648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878316.669833:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878316.669848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878316.669856:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878316/real 1788878316] req@ffffa06188698000 x1875771511483776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878316.669864:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511483776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878316.669870:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878316.669872:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878316.669874:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878316.669877:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878316.669888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878316.669893:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878316/real 1788878316] req@ffffa0618869ad80 x1875771511483904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878316.669901:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511483904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878316.669906:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878316.669907:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878316.669909:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511483904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878316.669913:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878316.669922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878316.669928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878316/real 1788878316] req@ffffa0618869bb80 x1875771511484032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878316.669939:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511484032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878316.669948:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878316.669951:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878316.669954:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878316.669961:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878316.669981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878316.669993:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878316/real 1788878316] req@ffffa0618869a680 x1875771511484160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878316.670004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511484160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878316.670012:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878316.670015:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878316.670019:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878317.309455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878317.309545:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878317.565434:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878318.333473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878318.333578:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878318.589501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878319.357476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878319.357584:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878319.613501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878320.381543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878320.381661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878320.637476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878321.405575:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878321.405672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878321.661492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878321.789580:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878321.789633:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878321.789639:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878321.789653:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3308 00000100:00080000:3.0:1788878321.789668:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878321.789695:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878321.789758:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511484288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878321.789802:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878321.789809:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878321.789812:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878321.789818:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3308 00000100:00080000:3.0:1788878321.789824:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878321.789830:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878321.789844:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878321.789846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511484288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878321.789865:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511484416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878321.789879:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878321.789882:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878321.789884:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878321.789888:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3308 00000100:00080000:3.0:1788878321.789892:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878321.789895:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878321.789900:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878321.789912:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511484544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878321.789936:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878321.789939:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878321.789941:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878321.789943:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3308 00000100:00080000:3.0:1788878321.789947:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878321.789950:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878321.789955:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878321.789957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878321.789977:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511484672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878321.789984:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878321.789988:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878321.789989:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878321.789992:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3308 00000100:00080000:3.0:1788878321.789995:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878321.789998:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878321.790001:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878321.790028:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511484800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878321.790053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878321/real 1788878321] req@ffffa06188871c00 x1875771511484288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878321.790091:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511484288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878321.790103:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878321.790108:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878321.790112:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511484288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878321.790155:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878321.790175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878321.790184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878321/real 1788878321] req@ffffa06188870380 x1875771511484416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878321.790199:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511484416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878321.790205:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878321.790207:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878321.790210:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878321.790214:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878321.790225:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878321.790231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878321/real 1788878321] req@ffffa06188870a80 x1875771511484544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878321.790239:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511484544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878321.790244:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878321.790246:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878321.790248:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878321.790251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878321.790263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878321.790269:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878321/real 1788878321] req@ffffa06188871880 x1875771511484672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878321.790277:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511484672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878321.790287:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878321.790289:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878321.790291:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878321.790295:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878321.790305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878321.790315:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878321/real 1788878321] req@ffffa06188872d80 x1875771511484800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878321.790328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511484800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878321.790337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878321.790341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878321.790344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511484800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878322.429452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878322.429652:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878322.685471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878323.453466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878323.453534:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878323.709453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878324.477464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878324.477552:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878324.733471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878325.501515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878325.501643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878325.757510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878326.525486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878326.525594:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878326.781630:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878326.909643:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878326.909659:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878326.909666:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878326.909687:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3313 00000100:00080000:3.0:1788878326.909710:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878326.909724:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878326.909792:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511484928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878326.909850:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878326.909859:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878326.909862:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878326.909868:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3313 00000100:00080000:3.0:1788878326.909875:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878326.909881:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878326.909896:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878326.909917:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511485056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878326.909927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511484928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878326.909958:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878326.909963:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878326.909965:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878326.909973:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3313 00000100:00080000:3.0:1788878326.909980:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878326.909985:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878326.909994:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878326.910014:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511485184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878326.910027:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878326.910042:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878326.910045:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878326.910050:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3313 00000100:00080000:3.0:1788878326.910055:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878326.910068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878326.910076:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878326.910088:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878326.910108:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511485312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878326.910125:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878326.910130:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878326.910133:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878326.910147:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3313 00000100:00100000:0.0:1788878326.910149:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878326/real 1788878326] req@ffffa06188872680 x1875771511484928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878326.910160:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878326.910184:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878326.910194:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878326.910211:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511485440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878326.910240:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511484928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878326.910265:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878326.910272:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878326.910277:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511484928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878326.910309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878326.910339:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878326.910373:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878326/real 1788878326] req@ffffa06188870700 x1875771511485056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878326.910389:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511485056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878326.910440:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878326.910444:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878326.910450:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878326.910465:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878326.910507:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878326.910545:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878326/real 1788878326] req@ffffa06188873b80 x1875771511485184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878326.910593:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511485184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878326.910603:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878326.910606:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878326.910610:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878326.910618:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878326.910651:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878326.910664:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878326/real 1788878326] req@ffffa06188872300 x1875771511485312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878326.910676:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511485312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878326.910684:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878326.910687:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878326.910691:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878326.910697:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878326.910713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878326.910723:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878326/real 1788878326] req@ffffa06188873480 x1875771511485440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878326.910733:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511485440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878326.910741:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878326.910744:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878326.910747:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878327.549619:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878327.549746:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878327.805445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878328.573551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878328.573627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878328.829492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878329.597522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878329.597662:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878329.853530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878330.621533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878330.621658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878330.877512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878331.645531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878331.645637:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878331.901515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878332.029492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878332.029502:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878332.029506:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878332.029515:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3318 00000100:00080000:3.0:1788878332.029537:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878332.029560:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878332.029594:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511485568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878332.029616:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878332.029619:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878332.029621:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878332.029624:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3318 00000100:00080000:3.0:1788878332.029627:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878332.029630:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878332.029637:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878332.029646:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511485696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878332.029654:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878332.029656:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878332.029657:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878332.029660:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3318 00000100:00080000:3.0:1788878332.029662:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878332.029664:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878332.029668:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878332.029674:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511485568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878332.029677:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511485824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878332.029690:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878332.029692:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878332.029693:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878332.029695:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3318 00000100:00080000:3.0:1788878332.029697:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878332.029699:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878332.029702:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878332.029709:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511485952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878332.029720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878332.029723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878332.029724:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878332.029726:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3318 00000100:00080000:3.0:1788878332.029729:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878332.029730:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878332.029733:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878332.029741:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511486080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878332.029752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878332.029791:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878332/real 1788878332] req@ffffa06188873800 x1875771511485568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878332.029809:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511485568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878332.029819:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878332.029823:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878332.029827:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511485568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878332.029833:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878332.029857:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878332.029870:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878332/real 1788878332] req@ffffa06188871500 x1875771511485696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878332.029882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511485696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878332.029890:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878332.029893:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878332.029897:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878332.029922:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878332.029945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878332.029956:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878332/real 1788878332] req@ffffa06188873100 x1875771511485824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878332.029970:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511485824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878332.029978:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878332.029981:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878332.029983:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878332.029987:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878332.030003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878332.030010:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878332/real 1788878332] req@ffffa06188872a00 x1875771511485952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878332.030017:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511485952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878332.030022:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878332.030024:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878332.030027:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511485952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878332.030030:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878332.030039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878332.030060:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878332/real 1788878332] req@ffffa06188873480 x1875771511486080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878332.030069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511486080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878332.030075:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878332.030077:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878332.030079:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878332.669523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878332.669605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878332.925497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878333.693486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878333.693568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878333.949584:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878334.717485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878334.717579:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878334.973475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878335.741465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878335.741537:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878335.997507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878336.765490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878336.765560:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878337.021513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878337.149530:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878337.149540:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878337.149545:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878337.149557:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3323 00000100:00080000:1.0:1788878337.149569:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878337.149577:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878337.149619:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511486208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878337.149645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878337.149648:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878337.149650:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878337.149655:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3323 00000100:00080000:1.0:1788878337.149660:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878337.149664:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878337.149675:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878337.149676:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511486208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878337.149695:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511486336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878337.149708:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878337.149711:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878337.149713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878337.149717:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3323 00000100:00080000:1.0:1788878337.149724:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878337.149728:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878337.149733:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788878337.149734:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878337.149749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511486464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878337.149758:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878337.149762:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878337.149764:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878337.149772:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878337/real 1788878337] req@ffffa0618869aa00 x1875771511486208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878337.149772:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3323 00000100:00080000:1.0:1788878337.149779:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878337.149789:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878337.149794:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511486208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878337.149796:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878337.149808:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878337.149812:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788878337.149818:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511486592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878337.149826:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511486208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878337.149842:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878337.149846:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878337.149849:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878337.149851:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486336:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788878337.149858:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3323 00000100:00080000:1.0:1788878337.149863:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878337.149868:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878337.149872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788878337.149880:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878337.149896:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878337/real 1788878337] req@ffffa0618869b800 x1875771511486336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878337.149908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511486336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878337.149914:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788878337.149916:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511486720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878337.149917:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878337.149923:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878337.149936:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878337.149961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878337.149974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878337/real 1788878337] req@ffffa06188699c00 x1875771511486464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878337.149986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511486464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878337.149994:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878337.149997:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878337.150000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878337.150006:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878337.150024:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878337.150034:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878337/real 1788878337] req@ffffa06188698380 x1875771511486592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878337.150051:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511486592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878337.150059:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878337.150062:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878337.150065:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878337.150081:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878337.150098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878337.150108:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878337/real 1788878337] req@ffffa0618869b100 x1875771511486720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878337.150126:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511486720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878337.150134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878337.150137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878337.150140:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878337.789525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878337.789569:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878338.045488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878338.813486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878338.813636:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878339.069504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878339.837488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878339.837522:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878340.093495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878340.861541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878340.861599:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878341.117564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878341.885469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878341.885549:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878342.141454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878342.269559:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878342.269572:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878342.269577:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878342.269590:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3328 00000100:00080000:1.0:1788878342.269602:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878342.269614:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878342.269653:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511486848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878342.269678:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878342.269681:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878342.269683:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878342.269687:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3328 00000100:00080000:1.0:1788878342.269690:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878342.269695:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878342.269704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878342.269716:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511486976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878342.269722:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511486848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878342.269733:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878342.269737:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878342.269738:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878342.269742:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3328 00000100:00080000:1.0:1788878342.269744:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878342.269748:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878342.269752:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878342.269763:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511487104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878342.269771:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878342.269773:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878342.269774:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878342.269777:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3328 00000100:00080000:1.0:1788878342.269782:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878342.269785:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878342.269789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878342.269794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878342.269802:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511487232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878342.269812:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878342.269815:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878342.269816:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878342.269822:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3328 00000100:00080000:1.0:1788878342.269825:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878342.269832:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878342.269833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878342/real 1788878342] req@ffffa0618869b100 x1875771511486848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878342.269836:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878342.269850:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511487360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878342.269858:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511486848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878342.269869:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878342.269873:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878342.269879:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511486848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878342.269899:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878342.269917:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878342.269927:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878342/real 1788878342] req@ffffa06188698380 x1875771511486976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878342.269936:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511486976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878342.269944:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878342.269946:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878342.269949:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511486976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878342.269954:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878342.269966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878342.269973:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878342/real 1788878342] req@ffffa06188699c00 x1875771511487104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878342.269999:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511487104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878342.270006:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878342.270008:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878342.270010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878342.270014:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878342.270028:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878342.270035:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878342/real 1788878342] req@ffffa0618869b800 x1875771511487232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878342.270044:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511487232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878342.270050:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878342.270052:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878342.270054:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878342.270059:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878342.270070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878342.270077:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878342/real 1788878342] req@ffffa0618869aa00 x1875771511487360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878342.270086:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511487360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878342.270097:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878342.270100:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878342.270102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878342.909521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878342.909605:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878343.165465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878343.933496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878343.933590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878344.189466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878344.957546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878344.957654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878345.213523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878345.981513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878345.981671:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878346.237521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878347.005579:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878347.005703:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878347.261542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878347.389596:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878347.389614:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878347.389621:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878347.389637:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3334 00000100:00080000:2.0:1788878347.389652:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878347.389664:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878347.389718:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511487488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878347.389776:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878347.389784:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878347.389788:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878347.389796:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3334 00000100:00080000:2.0:1788878347.389805:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878347.389814:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878347.389830:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878347.389847:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511487488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878347.389863:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511487616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878347.389887:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878347.389894:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878347.389898:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878347.389906:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3334 00000100:00080000:2.0:1788878347.389914:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878347.389922:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878347.389931:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878347.389938:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878347.389966:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511487744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878347.389982:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878347/real 1788878347] req@ffffa061888aca80 x1875771511487488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878347.389996:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878347.390008:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878347.390012:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878347.390015:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511487488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878347.390039:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3334 00000100:00080000:0.0:1788878347.390048:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878347.390049:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878347.390055:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788878347.390061:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878347.390065:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511487488:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878347.390071:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878347.390088:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878347.390093:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511487872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878347.390109:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878347.390114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878347.390117:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878347.390121:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878347.390127:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3334 00000100:00100000:0.0:1788878347.390130:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878347/real 1788878347] req@ffffa061888adc00 x1875771511487616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878347.390134:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878347.390145:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878347.390149:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511487616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878347.390153:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878347.390159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878347.390163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878347.390165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878347.390174:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878347.390175:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511488000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878347.390190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878347.390211:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878347/real 1788878347] req@ffffa061888ae300 x1875771511487744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878347.390223:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511487744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878347.390229:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878347.390231:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878347.390233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878347.390243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878347.390255:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878347.390262:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878347/real 1788878347] req@ffffa061888af480 x1875771511487872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878347.390270:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511487872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878347.390277:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878347.390279:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878347.390281:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511487872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878347.390285:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878347.390295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878347.390324:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878347/real 1788878347] req@ffffa061888ac000 x1875771511488000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878347.390336:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511488000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878347.390346:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878347.390349:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878347.390352:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878348.029543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878348.029599:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878348.285450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878349.053502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878349.053573:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878349.309505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878350.077485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878350.077558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878350.333536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878351.101493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878351.101548:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878351.357591:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878352.125541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878352.125672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878352.381575:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878352.509581:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878352.509595:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878352.509602:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878352.509619:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3339 00000100:00080000:3.0:1788878352.509637:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878352.509652:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878352.509709:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511488128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878352.509751:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878352.509756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878352.509759:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878352.509775:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3339 00000100:00080000:3.0:1788878352.509781:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878352.509787:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878352.509801:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878352.509824:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511488256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878352.509845:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878352.509851:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788878352.509853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511488128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878352.509856:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878352.509883:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3339 00000100:00080000:3.0:1788878352.509892:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878352.509902:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878352.509914:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878352.509940:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511488384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878352.509961:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878352.509965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878352.509968:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878352.509974:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878352.509981:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3339 00000100:00080000:3.0:1788878352.509989:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878352.509996:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878352.510003:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878352.510023:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511488512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878352.510036:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878352/real 1788878352] req@ffffa06188872300 x1875771511488128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878352.510043:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878352.510058:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878352.510065:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878352.510079:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3339 00000100:00080000:3.0:1788878352.510088:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878352.510101:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878352.510109:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878352.510119:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511488128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878352.510137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788878352.510141:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511488640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878352.510149:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878352.510166:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511488128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878352.510206:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878352.510239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878352.510256:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878352/real 1788878352] req@ffffa06188873b80 x1875771511488256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878352.510272:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511488256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878352.510283:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878352.510286:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878352.510291:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878352.510298:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878352.510318:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878352.510329:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878352/real 1788878352] req@ffffa06188870700 x1875771511488384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878352.510343:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511488384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878352.510377:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878352.510412:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878352.510416:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878352.510424:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878352.510479:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878352.510498:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878352/real 1788878352] req@ffffa06188872680 x1875771511488512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878352.510514:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511488512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878352.510524:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878352.510528:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878352.510532:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878352.510539:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878352.510563:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878352.510575:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878352/real 1788878352] req@ffffa06188872d80 x1875771511488640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878352.510589:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511488640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878352.510599:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878352.510603:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878352.510608:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878353.149537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878353.149669:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878353.405580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878354.173580:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878354.173687:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878354.429529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878355.197494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878355.197576:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878355.453500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878356.221535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878356.221653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878356.477605:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878357.245538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878357.245680:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878357.501556:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878357.629575:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878357.629604:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878357.629610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878357.629623:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3344 00000100:00080000:3.0:1788878357.629638:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878357.629648:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878357.629715:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511488768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878357.629751:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878357.629756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878357.629758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878357.629764:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3344 00000100:00080000:3.0:1788878357.629769:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878357.629774:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878357.629785:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878357.629802:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511488896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878357.629816:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511488768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878357.629841:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878357.629845:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878357.629848:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878357.629854:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3344 00000100:00080000:3.0:1788878357.629858:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878357.629862:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878357.629869:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878357.629895:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511489024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878357.629906:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878357.629910:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878357.629913:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878357.629922:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3344 00000800:00000100:0.0:1788878357.629928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878357.629936:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878357.629950:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878357.629961:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878357.629980:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878357/real 1788878357] req@ffffa06188871880 x1875771511488768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878357.630006:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511489152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878357.630009:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511488768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878357.630032:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878357.630041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878357.630042:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878357.630050:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511488768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878357.630052:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878357.630061:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878357.630070:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3344 00000100:00080000:3.0:1788878357.630077:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878357.630083:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878357.630090:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488896:192.168.203.111@tcp:38:kworker.0 00000008:00100000:3.0:1788878357.630096:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878357.630121:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511489280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878357.630142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878357.630175:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878357/real 1788878357] req@ffffa06188870a80 x1875771511488896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878357.630202:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511488896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878357.630215:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878357.630219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878357.630224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511488896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878357.630231:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878357.630260:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878357.630274:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878357/real 1788878357] req@ffffa06188870380 x1875771511489024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878357.630288:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511489024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878357.630297:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878357.630301:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878357.630304:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878357.630311:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878357.630329:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878357.630339:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878357/real 1788878357] req@ffffa06188871c00 x1875771511489152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878357.630352:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511489152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878357.630361:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878357.630364:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878357.630367:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878357.630419:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878357.630474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878357.630495:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878357/real 1788878357] req@ffffa06188871880 x1875771511489280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878357.630509:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511489280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878357.630519:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878357.630522:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878357.630526:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878358.269536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878358.269605:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878358.525570:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878359.293534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878359.293641:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878359.549526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878360.317504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878360.317622:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878360.573556:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878361.341531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878361.341670:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878361.597537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:0.0:1788878362.365614:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788878362.365654:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788878362.621501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878362.749564:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878362.749582:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878362.749591:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878362.749606:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3349 00000100:00080000:3.0:1788878362.749622:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878362.749633:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878362.749683:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511489408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878362.749723:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878362.749728:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878362.749730:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878362.749736:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3349 00000100:00080000:3.0:1788878362.749741:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878362.749746:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878362.749759:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878362.749778:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511489536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878362.749795:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878362.749798:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511489408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878362.749812:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878362.749818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878362.749824:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3349 00000100:00080000:3.0:1788878362.749828:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878362.749832:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878362.749838:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878362.749855:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511489664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878362.749868:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878362.749872:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878362.749875:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878362.749880:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3349 00000100:00080000:3.0:1788878362.749884:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878362.749888:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878362.749911:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878362.749927:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511489792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878362.749930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878362.749943:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878362.749947:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878362.749950:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878362.749955:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3349 00000100:00080000:3.0:1788878362.749959:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878362.749963:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878362.749968:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878362.749978:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511489920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878362.749991:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878362/real 1788878362] req@ffffa06188872d80 x1875771511489408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878362.750022:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511489408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878362.750039:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878362.750045:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878362.750051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511489408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878362.750098:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878362.750136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878362.750157:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878362/real 1788878362] req@ffffa06188872680 x1875771511489536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878362.750178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511489536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878362.750192:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878362.750197:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878362.750203:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878362.750214:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878362.750243:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878362.750261:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878362/real 1788878362] req@ffffa06188870700 x1875771511489664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878362.750280:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511489664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878362.750293:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878362.750298:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878362.750303:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878362.750312:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878362.750366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878362.750468:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878362/real 1788878362] req@ffffa06188873b80 x1875771511489792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878362.750497:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511489792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878362.750514:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878362.750521:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878362.750527:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878362.750540:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878362.750580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878362.750604:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878362/real 1788878362] req@ffffa06188872300 x1875771511489920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878362.750630:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511489920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878362.750647:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878362.750653:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878362.750660:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511489920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878363.389530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878363.389648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878363.645544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878364.413525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878364.413602:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878364.669577:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878365.437555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878365.437663:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878365.693531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878366.461526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878366.461650:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878366.717519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878367.485493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878367.485618:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878367.741524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878367.869546:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878367.869558:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878367.869564:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878367.869576:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3354 00000100:00080000:0.0:1788878367.869591:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878367.869600:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878367.869652:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511490048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878367.869705:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878367.869712:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878367.869716:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878367.869725:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3354 00000100:00080000:0.0:1788878367.869733:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878367.869745:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878367.869762:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878367.869778:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511490048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878367.869791:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511490176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878367.869809:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878367.869814:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878367.869817:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878367.869822:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3354 00000100:00080000:0.0:1788878367.869829:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878367.869834:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878367.869841:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878367.869857:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511490304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878367.869878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878367.869891:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878367.869896:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878367.869899:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878367.869906:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3354 00000100:00080000:0.0:1788878367.869911:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878367.869917:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878367.869924:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878367.869939:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511490432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878367.869953:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878367.869958:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878367/real 1788878367] req@ffffa06198afdf80 x1875771511490048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878367.869959:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878367.869966:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878367.869977:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3354 00000100:00080000:0.0:1788878367.869984:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878367.869996:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878367.870007:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878367.870032:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511490048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878367.870033:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511490560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878367.870105:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878367.870115:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878367.870121:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511490048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878367.870167:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878367.870219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878367.870241:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878367/real 1788878367] req@ffffa06198afdc00 x1875771511490176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878367.870267:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511490176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878367.870281:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878367.870287:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878367.870292:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878367.870335:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878367.870438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878367.870465:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878367/real 1788878367] req@ffffa06198afc700 x1875771511490304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878367.870483:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511490304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878367.870493:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878367.870498:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878367.870502:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878367.870510:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878367.870539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878367.870552:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878367/real 1788878367] req@ffffa06198afc000 x1875771511490432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878367.870567:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511490432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878367.870576:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878367.870580:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878367.870583:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878367.870590:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878367.870610:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878367.870621:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878367/real 1788878367] req@ffffa06198afea00 x1875771511490560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878367.870644:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511490560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878367.870655:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878367.870661:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878367.870667:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878368.509485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878368.509548:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878368.765496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878369.533475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878369.533549:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878369.789521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878370.557514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878370.557578:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878370.813516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878371.581508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878371.581620:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878371.837618:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878372.605568:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878372.605627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878372.861525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878372.989582:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878372.989596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878372.989603:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878372.989618:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3359 00000100:00080000:3.0:1788878372.989635:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878372.989648:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878372.989699:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511490688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878372.989730:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878372.989735:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878372.989738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878372.989743:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3359 00000100:00080000:3.0:1788878372.989749:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878372.989766:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878372.989779:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878372.989797:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511490688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878372.989819:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511490816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878372.989852:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878372.989857:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878372.989860:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878372.989865:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3359 00000100:00080000:3.0:1788878372.989869:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878372.989874:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878372.989881:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878372.989897:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511490944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878372.989909:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878372.989914:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878372.989917:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878372.989922:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3359 00000100:00080000:3.0:1788878372.989926:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878372.989929:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878372.989936:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878372.989953:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511491072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878372.989965:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878372.989970:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878372.989973:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878372.989978:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3359 00000100:00080000:3.0:1788878372.989985:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878372.989989:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490816:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788878372.989994:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878372.990007:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878372.990028:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511491200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878372.990077:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878372.990123:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878372.990173:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878373.629578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878373.629699:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878373.885633:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878374.653588:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878374.653729:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878374.909516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878375.677533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878375.677651:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878375.933568:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878376.061802:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878376.061873:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878376.061900:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878376.061969:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878376.061987:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878376.062014:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878376.062032:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878376.062069:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878372/real 1788878376] req@ffffa06188873800 x1875771511491200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878376.062120:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511491200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878376.062141:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878376.062149:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878376.062157:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878376.062172:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878372/real 1788878376] req@ffffa06188872a00 x1875771511490816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878376.062198:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511490816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878376.062228:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878376.062235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878376.062243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878376.062254:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878372/real 1788878376] req@ffffa06188873100 x1875771511490944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878376.062277:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511490944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878376.062291:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878376.062296:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878376.062302:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511490944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878376.062313:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878372/real 1788878376] req@ffffa06188871500 x1875771511491072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878376.062332:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511491072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878376.062376:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878376.062422:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878376.062431:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878376.062445:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878372/real 1788878376] req@ffffa06188873480 x1875771511490688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878376.062472:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511490688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878376.062514:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878376.062521:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878376.062529:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511490688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878376.701603:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878376.701625:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878376.957587:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878377.725511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878377.725590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878377.981553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878378.109597:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878378.109611:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878378.109617:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878378.109632:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3364 00000100:00080000:2.0:1788878378.109649:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878378.109660:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878378.109725:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511491328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878378.109775:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878378.109783:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878378.109787:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878378.109796:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3364 00000100:00080000:2.0:1788878378.109804:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878378.109813:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878378.109830:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878378.109864:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511491328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878378.109873:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511491456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878378.109903:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878378.109910:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878378.109924:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878378.109931:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3364 00000100:00080000:2.0:1788878378.109939:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878378.109949:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878378.109960:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878378.109997:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511491584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878378.110017:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878378.110018:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878378.110030:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878378.110036:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878378.110046:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3364 00000100:00080000:2.0:1788878378.110062:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878378.110072:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878378.110092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878378/real 1788878378] req@ffffa061888ad500 x1875771511491328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878378.110106:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878378.110155:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511491328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878378.110170:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511491712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878378.110193:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878378.110204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878378.110213:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511491328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878378.110241:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878378.110257:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878378.110262:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878378.110272:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3364 00000100:00100000:0.0:1788878378.110279:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491456:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878378.110284:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878378.110300:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878378.110313:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878378.110362:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511491840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878378.110365:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878378.110493:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878378/real 1788878378] req@ffffa061888adf80 x1875771511491456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878378.110516:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511491456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878378.110540:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878378.110548:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878378.110553:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878378.110564:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878378.110602:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878378.110623:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878378/real 1788878378] req@ffffa061888ac000 x1875771511491584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878378.110644:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511491584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878378.110657:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878378.110662:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878378.110667:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878378.110677:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878378.110710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878378.110739:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878378/real 1788878378] req@ffffa061888af480 x1875771511491712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878378.110759:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511491712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878378.110772:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878378.110778:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878378.110783:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878378.110814:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878378.110849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878378.110869:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878378/real 1788878378] req@ffffa061888ae300 x1875771511491840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878378.110889:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511491840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878378.110903:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878378.110908:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878378.110914:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511491840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878378.749489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878378.749570:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878379.005522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878379.773471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878379.773538:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878380.029533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878380.797464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878380.797530:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878381.053458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878381.821545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878381.821619:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878382.077530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878382.845452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878382.845514:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878383.101504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878383.229514:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878383.229524:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878383.229553:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878383.229563:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3369 00000100:00080000:2.0:1788878383.229575:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878383.229582:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878383.229620:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511491968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878383.229646:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878383.229649:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878383.229652:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878383.229656:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3369 00000100:00080000:2.0:1788878383.229660:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878383.229663:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878383.229672:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878383.229682:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511491968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878383.229685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511492096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878383.229699:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878383.229702:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878383.229703:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878383.229706:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3369 00000100:00080000:2.0:1788878383.229709:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878383.229711:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878383.229715:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878383.229726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511492224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878383.229733:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878383.229736:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878383.229738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878383.229741:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3369 00000100:00080000:2.0:1788878383.229744:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878383.229748:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878383.229753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878383.229754:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878383.229766:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511492352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878383.229775:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878383.229778:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878383.229780:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878383.229784:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3369 00000100:00080000:2.0:1788878383.229787:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878383.229792:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878383.229795:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878383/real 1788878383] req@ffffa061888adc00 x1875771511491968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878383.229797:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878383.229811:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511492480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878383.229821:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511491968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878383.229833:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878383.229838:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878383.229842:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511491968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878383.229866:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878383.229883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878383.229892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878383/real 1788878383] req@ffffa061888aca80 x1875771511492096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878383.229902:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511492096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878383.229909:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878383.229912:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878383.229915:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878383.229919:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878383.229931:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878383.229939:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878383/real 1788878383] req@ffffa061888afb80 x1875771511492224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878383.229948:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511492224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878383.229954:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878383.229956:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878383.229959:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878383.229963:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878383.229993:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878383.230002:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878383/real 1788878383] req@ffffa061888af100 x1875771511492352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878383.230013:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511492352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878383.230020:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878383.230022:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878383.230025:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878383.230030:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878383.230060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878383.230068:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878383/real 1788878383] req@ffffa061888ae680 x1875771511492480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878383.230077:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511492480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878383.230083:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878383.230086:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878383.230090:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878383.869493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878383.869561:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878384.125531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878384.893465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878384.893525:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878385.149459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878385.917507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878385.917661:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878386.173536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878386.941531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878386.941653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878387.197471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878387.965568:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878387.965668:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878388.221485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878388.349501:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878388.349513:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878388.349518:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878388.349531:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3375 00000100:00080000:2.0:1788878388.349545:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878388.349555:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878388.349599:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511492608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878388.349645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878388.349650:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878388.349653:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878388.349658:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3375 00000100:00080000:2.0:1788878388.349664:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878388.349669:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878388.349680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878388.349695:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511492608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878388.349700:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511492736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878388.349722:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878388.349726:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878388.349728:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878388.349744:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3375 00000100:00080000:2.0:1788878388.349747:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878388.349750:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878388.349755:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878388.349771:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511492864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878388.349781:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878388.349785:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878388.349788:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878388.349793:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3375 00000100:00080000:2.0:1788878388.349797:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878388.349806:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878388.349806:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878388.349817:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878388.349829:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511492992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878388.349838:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878388.349841:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878388.349843:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878388.349849:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3375 00000100:00100000:0.0:1788878388.349862:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878388/real 1788878388] req@ffffa061888ac380 x1875771511492608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878388.349864:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878388.349874:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878388.349882:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878388.349917:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511493120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878388.349923:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511492608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878388.349952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878388.349956:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878388.349961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511492608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878388.350003:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878388.350040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878388.350054:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878388/real 1788878388] req@ffffa061888aea00 x1875771511492736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878388.350066:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511492736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878388.350074:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878388.350078:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878388.350081:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878388.350086:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878388.350101:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878388.350109:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878388/real 1788878388] req@ffffa061888ad880 x1875771511492864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878388.350127:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511492864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878388.350134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878388.350136:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878388.350139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878388.350145:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878388.350159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878388.350167:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878388/real 1788878388] req@ffffa061888aed80 x1875771511492992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878388.350178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511492992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878388.350185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878388.350187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878388.350190:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511492992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878388.350195:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878388.350208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878388.350231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878388/real 1788878388] req@ffffa061888ac700 x1875771511493120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878388.350241:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511493120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878388.350255:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878388.350258:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878388.350261:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878388.989510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878388.989761:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878389.245486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878390.013456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878390.013518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878390.269471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878391.037515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878391.037621:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878391.293485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878392.061485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878392.061553:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878392.317492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878393.085471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878393.085548:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878393.341455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878393.469485:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878393.469493:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878393.469497:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878393.469507:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3380 00000100:00080000:2.0:1788878393.469516:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878393.469522:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878393.469555:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511493248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878393.469599:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878393.469602:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878393.469603:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878393.469607:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3380 00000100:00080000:2.0:1788878393.469610:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878393.469614:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878393.469623:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878393.469633:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511493248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878393.469639:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511493376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878393.469648:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878393.469651:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878393.469652:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878393.469655:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3380 00000100:00080000:2.0:1788878393.469657:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878393.469660:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878393.469664:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878393.469673:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511493504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878393.469680:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878393.469683:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878393.469685:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878393.469687:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3380 00000100:00080000:2.0:1788878393.469691:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878393.469693:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878393.469698:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878393.469699:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878393.469710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511493632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878393.469718:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878393.469720:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878393.469722:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878393.469726:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3380 00000100:00080000:2.0:1788878393.469729:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878393.469734:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878393.469735:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878393/real 1788878393] req@ffffa061888ad180 x1875771511493248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878393.469738:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878393.469751:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511493760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878393.469759:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511493248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878393.469769:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878393.469774:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878393.469778:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511493248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878393.469799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878393.469815:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878393.469824:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878393/real 1788878393] req@ffffa061888ace00 x1875771511493376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878393.469838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511493376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878393.469845:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878393.469847:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878393.469850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878393.469854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878393.469865:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878393.469872:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878393/real 1788878393] req@ffffa061888ac700 x1875771511493504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878393.469880:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511493504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878393.469885:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878393.469887:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878393.469889:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878393.469894:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878393.469905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878393.469912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878393/real 1788878393] req@ffffa061888aed80 x1875771511493632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878393.469920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511493632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878393.469931:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878393.469933:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878393.469936:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878393.469939:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878393.469949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878393.469971:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878393/real 1788878393] req@ffffa061888ad880 x1875771511493760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878393.469978:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511493760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878393.469983:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878393.469985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878393.469987:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511493760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878394.109530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878394.109587:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878394.365494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878395.133533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878395.133620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878395.389505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878396.157552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878396.157619:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878396.413518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878397.181489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878397.181609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878397.437526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878398.205526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878398.205614:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878398.461506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878398.589560:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878398.589577:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878398.589585:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878398.589604:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3385 00000100:00080000:2.0:1788878398.589620:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878398.589631:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878398.589682:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511493888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878398.589717:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878398.589721:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878398.589724:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878398.589729:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3385 00000100:00080000:2.0:1788878398.589733:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878398.589739:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878398.589752:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878398.589769:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511494016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878398.589779:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511493888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878398.589804:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878398.589811:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878398.589816:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878398.589824:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3385 00000100:00080000:2.0:1788878398.589833:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878398.589842:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878398.589853:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878398.589872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878398.589885:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511494144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878398.589910:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878398.589918:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878398.589922:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878398.589925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878398/real 1788878398] req@ffffa061888aea00 x1875771511493888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878398.589934:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3385 00000100:00080000:2.0:1788878398.589946:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878398.589957:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878398.589967:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878398.589968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511493888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878398.589989:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788878398.589991:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511494272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878398.589999:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878398.590015:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511493888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878398.590023:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878398.590028:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878398.590031:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878398.590038:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3385 00000100:00080000:2.0:1788878398.590041:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878398.590046:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878398.590054:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494016:192.168.203.111@tcp:38:kworker.0 00000008:00100000:2.0:1788878398.590055:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878398.590075:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511494400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878398.590120:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878398.590140:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878398/real 1788878398] req@ffffa061888ac380 x1875771511494016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878398.590156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511494016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878398.590167:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878398.590171:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878398.590176:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878398.590183:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878398.590203:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878398.590215:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878398/real 1788878398] req@ffffa061888ae680 x1875771511494144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878398.590230:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511494144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878398.590270:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878398.590275:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878398.590279:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878398.590287:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878398.590310:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878398.590324:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878398/real 1788878398] req@ffffa061888af100 x1875771511494272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878398.590338:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511494272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878398.590348:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878398.590351:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878398.590355:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878398.590375:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878398.590428:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878398.590442:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878398/real 1788878398] req@ffffa061888afb80 x1875771511494400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878398.590457:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511494400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878398.590467:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878398.590471:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878398.590476:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878399.229565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878399.229683:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878399.485476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878400.253482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878400.253566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878400.509462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878401.277535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878401.277631:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878401.533552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878402.301538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878402.301690:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878402.557456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878403.325562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878403.325605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878403.581473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878403.709510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878403.709530:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878403.709535:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878403.709548:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3390 00000100:00080000:1.0:1788878403.709559:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878403.709566:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878403.709621:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511494528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878403.709664:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878403.709669:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878403.709672:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878403.709679:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3390 00000100:00080000:1.0:1788878403.709684:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878403.709690:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878403.709698:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511494528:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878403.709712:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878403.709736:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511494656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878403.709751:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878403.709755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878403.709757:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878403.709762:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878403.709768:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3390 00000100:00080000:1.0:1788878403.709775:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878403.709779:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878403.709785:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878403.709814:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878403/real 1788878403] req@ffffa0618869a680 x1875771511494528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878403.709815:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511494784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878403.709857:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878403.709861:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878403.709862:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878403.709875:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511494528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878403.709876:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3390 00000100:00080000:1.0:1788878403.709884:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878403.709891:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878403.709894:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788878403.709897:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878403.709902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878403.709907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511494528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878403.709914:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511494912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878403.709922:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878403.709925:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878403.709927:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878403.709930:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3390 00000100:00080000:1.0:1788878403.709934:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878403.709936:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494656:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788878403.709941:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878403.709949:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878403.709961:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511495040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878403.709970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878403.709986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878403/real 1788878403] req@ffffa0618869bb80 x1875771511494656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878403.710004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511494656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878403.710013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878403.710016:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878403.710018:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878403.710023:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878403.710036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878403.710044:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878403/real 1788878403] req@ffffa0618869ad80 x1875771511494784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878403.710064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511494784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878403.710070:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878403.710072:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878403.710075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878403.710079:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878403.710104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878403.710113:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878403/real 1788878403] req@ffffa06188698000 x1875771511494912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878403.710122:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511494912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878403.710129:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878403.710136:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878403.710139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511494912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878403.710153:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878403.710167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878403.710175:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878403/real 1788878403] req@ffffa06188699f80 x1875771511495040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878403.710184:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511495040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878403.710191:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878403.710194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878403.710196:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878404.349495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878404.349522:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878404.605456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878405.373509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878405.373606:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878405.629553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878406.397493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878406.397644:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878406.653513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878407.421518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878407.421643:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878407.677486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878408.445522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878408.445702:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878408.701513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878408.829459:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878408.829470:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878408.829475:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878408.829487:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3395 00000100:00080000:1.0:1788878408.829501:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878408.829509:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878408.829559:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511495168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878408.829589:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878408.829593:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878408.829595:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878408.829599:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3395 00000100:00080000:1.0:1788878408.829603:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878408.829608:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878408.829618:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878408.829630:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511495168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878408.829632:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511495296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878408.829654:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878408.829657:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878408.829658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878408.829663:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3395 00000100:00080000:1.0:1788878408.829667:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878408.829669:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878408.829674:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878408.829683:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511495424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878408.829690:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878408.829694:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878408.829695:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878408.829698:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3395 00000100:00080000:1.0:1788878408.829709:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878408.829715:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878408.829720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788878408.829726:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878408.829742:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511495552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878408.829751:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878408.829754:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878408.829756:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878408.829759:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3395 00000100:00080000:1.0:1788878408.829761:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878408.829763:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878408.829766:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878408.829770:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878408/real 1788878408] req@ffffa06188699f80 x1875771511495168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878408.829780:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511495680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878408.829832:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511495168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878408.829842:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878408.829847:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878408.829852:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511495168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878408.829870:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878408.829886:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878408.829894:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878408/real 1788878408] req@ffffa06188698000 x1875771511495296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878408.829902:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511495296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878408.829908:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878408.829911:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878408.829913:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878408.829917:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878408.829927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878408.829933:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878408/real 1788878408] req@ffffa0618869ad80 x1875771511495424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878408.829940:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511495424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878408.829949:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878408.829951:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878408.829953:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878408.829957:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878408.829968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878408.829974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878408/real 1788878408] req@ffffa0618869bb80 x1875771511495552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878408.829981:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511495552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878408.829986:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878408.829989:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878408.829990:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878408.829994:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878408.830004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878408.830010:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878408/real 1788878408] req@ffffa0618869a680 x1875771511495680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878408.830017:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511495680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878408.830022:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878408.830024:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878408.830027:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878409.469516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878409.469565:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878409.725476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878410.493496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878410.493639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878410.749488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878411.517476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878411.517576:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878411.773501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878412.541556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878412.541637:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878412.797439:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878413.565468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878413.565597:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878413.821476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878413.949516:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878413.949527:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878413.949531:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878413.949542:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3400 00000100:00080000:2.0:1788878413.949555:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878413.949564:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878413.949621:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511495808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878413.949642:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878413.949645:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878413.949646:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878413.949649:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3400 00000100:00080000:2.0:1788878413.949652:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878413.949655:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878413.949663:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878413.949671:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511495808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878413.949674:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511495936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878413.949685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878413.949687:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878413.949688:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878413.949690:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3400 00000100:00080000:2.0:1788878413.949692:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878413.949694:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878413.949698:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878413.949707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511496064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878413.949713:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878413.949715:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878413.949717:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878413.949721:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3400 00000100:00080000:2.0:1788878413.949723:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878413.949726:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878413.949729:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878413.949731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878413.949740:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511496192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878413.949747:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878413.949749:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878413.949750:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878413.949753:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3400 00000100:00080000:2.0:1788878413.949755:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878413.949758:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878413.949762:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878413.949766:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878413/real 1788878413] req@ffffa061888aca80 x1875771511495808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878413.949771:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511496320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878413.949797:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511495808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878413.949807:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878413.949810:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878413.949813:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511495808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878413.949832:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878413.949868:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878413.949880:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878413/real 1788878413] req@ffffa061888adc00 x1875771511495936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878413.949892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511495936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878413.949901:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878413.949905:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878413.949908:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511495936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878413.949913:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878413.949928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878413.949937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878413/real 1788878413] req@ffffa061888ae300 x1875771511496064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878413.949947:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511496064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878413.949955:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878413.949958:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878413.949961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878413.949966:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878413.949981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878413.949994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878413/real 1788878413] req@ffffa061888af480 x1875771511496192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878413.950013:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511496192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878413.950026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878413.950030:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878413.950036:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878413.950046:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878413.950077:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878413.950095:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878413/real 1788878413] req@ffffa061888ac000 x1875771511496320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878413.950114:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511496320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878413.950127:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878413.950132:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878413.950138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878414.589492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878414.589628:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878414.845496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878415.613499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878415.613567:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878415.869473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878416.637504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878416.637620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878416.893517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878417.661520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878417.661675:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878417.917517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878418.685521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878418.685690:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878418.941620:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878419.069540:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878419.069551:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878419.069556:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878419.069568:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3405 00000100:00080000:2.0:1788878419.069581:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878419.069591:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878419.069632:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511496448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878419.069665:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878419.069670:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878419.069691:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878419.069698:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3405 00000100:00080000:2.0:1788878419.069703:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878419.069708:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878419.069719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878419.069743:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511496576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878419.069745:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511496448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878419.069780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878419.069784:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878419.069786:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878419.069791:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3405 00000100:00080000:2.0:1788878419.069794:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878419.069798:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878419.069804:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878419.069817:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511496704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878419.069828:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878419.069833:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878419.069836:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878419.069841:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3405 00000100:00080000:2.0:1788878419.069845:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878419.069850:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878419.069856:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878419.069867:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511496832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878419.069877:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878419.069884:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788878419.069889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878419.069889:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878419.069902:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3405 00000100:00080000:2.0:1788878419.069906:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878419.069911:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878419.069916:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878419.069931:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511496960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878419.069959:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878419/real 1788878419] req@ffffa061888adf80 x1875771511496448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878419.069986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511496448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878419.069999:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878419.070005:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878419.070008:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511496448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878419.070035:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878419.070057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878419.070076:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878419/real 1788878419] req@ffffa061888ad500 x1875771511496576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878419.070085:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511496576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878419.070092:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878419.070094:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878419.070097:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878419.070101:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878419.070113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878419.070119:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878419/real 1788878419] req@ffffa061888ac000 x1875771511496704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878419.070127:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511496704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878419.070152:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878419.070155:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878419.070158:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878419.070164:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878419.070192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878419.070200:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878419/real 1788878419] req@ffffa061888af480 x1875771511496832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878419.070208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511496832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878419.070214:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878419.070216:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878419.070219:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878419.070222:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878419.070239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878419.070247:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878419/real 1788878419] req@ffffa061888ae300 x1875771511496960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878419.070255:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511496960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878419.070260:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878419.070262:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878419.070265:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511496960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878419.709527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878419.709706:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878419.965548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878420.733562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878420.733716:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878420.989501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878421.757520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878421.757618:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878422.013492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878422.781502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878422.781604:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878423.037581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878423.805533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878423.805692:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878424.061551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878424.189518:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878424.189536:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878424.189543:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878424.189559:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3410 00000100:00080000:2.0:1788878424.189574:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878424.189585:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878424.189629:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511497088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878424.189667:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878424.189676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878424.189679:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878424.189685:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3410 00000100:00080000:2.0:1788878424.189691:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878424.189698:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878424.189711:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878424.189723:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511497088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878424.189732:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511497216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878424.189751:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878424.189755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878424.189757:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878424.189764:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3410 00000100:00080000:2.0:1788878424.189768:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878424.189788:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878424.189792:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878424.189801:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511497344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878424.189808:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878424.189811:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878424.189812:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878424.189815:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3410 00000100:00080000:2.0:1788878424.189818:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878424.189820:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878424.189824:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878424.189831:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878424.189835:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511497472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878424.189866:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878424.189872:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878424.189874:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878424.189880:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3410 00000100:00080000:2.0:1788878424.189886:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878424.189898:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878424.189904:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878424.189917:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511497600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878424.189938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878424/real 1788878424] req@ffffa061888adc00 x1875771511497088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878424.189969:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511497088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878424.189984:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878424.189989:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878424.189995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511497088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878424.190026:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878424.190058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878424.190074:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878424/real 1788878424] req@ffffa061888aca80 x1875771511497216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878424.190100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511497216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878424.190140:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878424.190145:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878424.190165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878424.190169:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878424.190191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878424.190200:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878424/real 1788878424] req@ffffa061888afb80 x1875771511497344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878424.190216:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511497344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878424.190243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878424.190247:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878424.190251:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878424.190258:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878424.190282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878424.190293:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878424/real 1788878424] req@ffffa061888af100 x1875771511497472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878424.190313:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511497472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878424.190323:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878424.190326:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878424.190331:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878424.190338:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878424.190356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878424.190366:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878424/real 1788878424] req@ffffa061888ae680 x1875771511497600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878424.190395:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511497600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878424.190401:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878424.190402:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878424.190404:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878424.829501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878424.829621:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878425.085567:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878425.853527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878425.853612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878426.109516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878426.877521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878426.877617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878427.133581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878427.901583:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878427.901683:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878428.157507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878428.925534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878428.925650:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878429.181557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878429.309557:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878429.309571:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878429.309578:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878429.309594:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3416 00000100:00080000:2.0:1788878429.309608:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878429.309620:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878429.309673:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511497728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878429.309724:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878429.309730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878429.309732:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878429.309738:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3416 00000100:00080000:2.0:1788878429.309743:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878429.309749:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878429.309762:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878429.309782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511497856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878429.309798:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511497728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878429.309801:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878429.309813:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878429.309820:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878429.309825:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3416 00000100:00080000:2.0:1788878429.309828:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878429.309832:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878429.309839:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878429.309854:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511497984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878429.309867:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878429.309871:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878429.309873:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878429.309878:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3416 00000100:00080000:2.0:1788878429.309882:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878429.309886:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878429.309891:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878429.309896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878429.309924:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511498112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878429.309945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878429/real 1788878429] req@ffffa061888ac380 x1875771511497728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878429.309952:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878429.309966:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878429.309972:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878429.309985:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3416 00000100:00080000:2.0:1788878429.309994:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878429.310004:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878429.310012:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878429.310021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511497728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878429.310029:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511498240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878429.310053:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878429.310063:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878429.310094:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511497728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878429.310129:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878429.310162:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878429.310177:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878429/real 1788878429] req@ffffa061888aea00 x1875771511497856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878429.310192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511497856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878429.310203:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878429.310207:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878429.310211:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878429.310218:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878429.310236:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878429.310247:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878429/real 1788878429] req@ffffa061888ad880 x1875771511497984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878429.310260:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511497984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878429.310277:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878429.310280:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878429.310284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511497984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878429.310290:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878429.310307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878429.310316:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878429/real 1788878429] req@ffffa061888aed80 x1875771511498112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878429.310329:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511498112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878429.310358:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878429.310361:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878429.310365:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878429.310373:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878429.310392:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878429.310438:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878429/real 1788878429] req@ffffa061888ac700 x1875771511498240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878429.310453:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511498240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878429.310463:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878429.310467:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878429.310472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878429.949466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878429.949692:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878430.205491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878430.973532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878430.973612:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878431.229519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878431.997586:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878431.997709:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878432.253530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878433.021480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878433.021629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878433.277491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878434.045522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878434.045664:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878434.301500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878434.429541:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878434.429551:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878434.429555:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878434.429564:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3421 00000100:00080000:2.0:1788878434.429590:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878434.429600:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878434.429650:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511498368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878434.429682:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878434.429686:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878434.429688:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878434.429694:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3421 00000100:00080000:2.0:1788878434.429699:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878434.429704:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878434.429730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878434.429738:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511498368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878434.429747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511498496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878434.429757:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878434.429761:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878434.429763:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878434.429766:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3421 00000100:00080000:2.0:1788878434.429769:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878434.429772:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878434.429776:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878434.429786:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511498624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878434.429794:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878434.429797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878434.429799:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878434.429805:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3421 00000800:00000100:0.0:1788878434.429806:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878434.429809:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878434.429814:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878434.429818:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878434.429830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511498752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878434.429837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878434.429840:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878434.429842:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878434.429847:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878434/real 1788878434] req@ffffa061888ace00 x1875771511498368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878434.429850:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3421 00000100:00080000:2.0:1788878434.429855:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878434.429860:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878434.429863:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878434.429872:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511498880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878434.429900:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511498368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878434.429917:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878434.429921:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878434.429924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511498368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878434.429946:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878434.429969:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878434.429980:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878434/real 1788878434] req@ffffa061888ad180 x1875771511498496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878434.429989:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511498496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878434.429997:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878434.429999:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878434.430002:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878434.430006:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878434.430019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878434.430026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878434/real 1788878434] req@ffffa061888ac700 x1875771511498624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878434.430054:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511498624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878434.430065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878434.430069:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878434.430073:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878434.430080:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878434.430106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878434.430122:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878434/real 1788878434] req@ffffa061888aed80 x1875771511498752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878434.430136:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511498752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878434.430142:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878434.430145:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878434.430148:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878434.430153:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878434.430169:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878434.430177:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878434/real 1788878434] req@ffffa061888ad880 x1875771511498880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878434.430186:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511498880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878434.430192:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878434.430194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878434.430196:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511498880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878435.069486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878435.069569:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878435.325446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878436.093492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878436.093555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878436.349476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878437.117472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878437.117539:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878437.373452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878438.141523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878438.141635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878438.397513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878439.165508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878439.165698:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878439.421472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878439.549492:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878439.549503:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878439.549508:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878439.549522:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3426 00000100:00080000:2.0:1788878439.549534:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878439.549542:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878439.549588:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511499008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878439.549626:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878439.549633:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878439.549636:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878439.549644:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3426 00000100:00080000:2.0:1788878439.549652:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878439.549658:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878439.549670:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511499008:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878439.549670:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878439.549694:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511499136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878439.549713:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878439.549718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878439.549720:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878439.549724:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3426 00000100:00080000:2.0:1788878439.549729:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878439.549733:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878439.549737:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878439.549749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511499264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878439.549773:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878439.549777:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878439.549778:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878439.549782:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3426 00000100:00080000:2.0:1788878439.549785:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878439.549789:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878439.549793:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878439.549805:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511499392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878439.549813:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878439.549817:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878439.549819:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878439.549825:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499136:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878439.549826:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3426 00000100:00080000:2.0:1788878439.549833:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878439.549841:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878439.549846:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878439.549858:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511499520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878439.549888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878439.549928:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878439.549966:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878440.189472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878440.189518:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878440.445500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878441.213435:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878441.213547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878441.469442:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878442.237469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878442.237579:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878442.493551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878442.621633:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878442.621670:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878442.621684:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878442.621713:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878442.621722:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878442.621730:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878442.621737:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878442.621746:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878439/real 1788878442] req@ffffa061888ac380 x1875771511499136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878442.621767:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511499136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878442.621777:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878442.621780:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878442.621784:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878442.621789:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878439/real 1788878442] req@ffffa061888ae680 x1875771511499264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878442.621796:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511499264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878442.621801:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878442.621803:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878442.621805:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878442.621810:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878439/real 1788878442] req@ffffa061888af100 x1875771511499392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878442.621822:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511499392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878442.621827:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878442.621829:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878442.621831:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878442.621835:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878439/real 1788878442] req@ffffa061888aea00 x1875771511499008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878442.621842:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511499008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878442.621846:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878442.621848:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878442.621850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511499008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878442.621873:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878439/real 1788878442] req@ffffa061888afb80 x1875771511499520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878442.621885:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511499520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878442.621893:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878442.621896:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878442.621898:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878443.261465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878443.261563:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878443.517476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878444.285477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878444.285571:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878444.541496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878444.669544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878444.669554:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878444.669560:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878444.669572:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3431 00000100:00080000:3.0:1788878444.669585:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878444.669594:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878444.669652:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511499648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878444.669710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878444.669718:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878444.669721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878444.669729:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3431 00000100:00080000:3.0:1788878444.669739:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878444.669748:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878444.669777:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878444.669779:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511499648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878444.669803:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511499776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878444.669837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878444.669844:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878444.669846:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878444.669850:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3431 00000100:00080000:3.0:1788878444.669856:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878444.669861:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878444.669867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788878444.669868:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878444.669902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511499904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878444.669912:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878444.669917:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878444/real 1788878444] req@ffffa06188873480 x1875771511499648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878444.669918:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878444.669924:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878444.669934:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3431 00000100:00080000:3.0:1788878444.669939:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878444.669943:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511499648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878444.669945:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878444.669953:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878444.669981:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788878444.669988:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511500032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878444.669989:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878444.670003:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878444.670008:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511499648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878444.670010:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878444.670014:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878444.670022:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3431 00000100:00080000:3.0:1788878444.670027:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878444.670032:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878444.670039:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878444.670043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878444.670055:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511500160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878444.670076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878444.670094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878444/real 1788878444] req@ffffa06188871500 x1875771511499776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878444.670106:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511499776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878444.670113:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878444.670117:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878444.670120:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878444.670127:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878444.670149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878444.670160:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878444/real 1788878444] req@ffffa06188873100 x1875771511499904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878444.670172:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511499904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878444.670180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878444.670183:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878444.670186:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511499904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878444.670192:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878444.670211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878444.670222:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878444/real 1788878444] req@ffffa06188872a00 x1875771511500032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878444.670232:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511500032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878444.670240:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878444.670243:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878444.670245:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878444.670264:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878444.670282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878444.670292:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878444/real 1788878444] req@ffffa06188873800 x1875771511500160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878444.670310:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511500160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878444.670317:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878444.670320:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878444.670323:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878445.309492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878445.309538:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878445.565488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878446.333501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878446.333627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878446.589534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878447.357518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878447.357603:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878447.613484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878448.381507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878448.381644:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878448.637506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878449.405505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878449.405610:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878449.661447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878449.789474:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878449.789485:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878449.789490:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878449.789501:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3436 00000100:00080000:3.0:1788878449.789513:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878449.789521:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878449.789560:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511500288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878449.789589:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878449.789594:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878449.789596:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878449.789602:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3436 00000100:00080000:3.0:1788878449.789607:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878449.789613:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878449.789623:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511500288:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878449.789623:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878449.789643:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511500416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878449.789656:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878449.789660:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878449.789662:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878449.789667:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3436 00000100:00080000:3.0:1788878449.789672:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878449.789677:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878449.789683:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878449.789685:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878449.789701:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511500544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878449.789714:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878449.789717:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878449/real 1788878449] req@ffffa06188872300 x1875771511500288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878449.789719:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878449.789724:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878449.789733:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3436 00000100:00100000:0.0:1788878449.789738:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511500288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878449.789740:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878449.789746:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878449.789759:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878449.789764:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878449.789767:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511500288:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878449.789769:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878449.789785:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511500672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878449.789789:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500416:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788878449.789806:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878449.789811:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788878449.789814:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878449.789816:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878449.789824:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3436 00000100:00100000:0.0:1788878449.789829:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878449/real 1788878449] req@ffffa06188873b80 x1875771511500416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878449.789831:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878449.789841:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878449.789846:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511500416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878449.789850:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878449.789855:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878449.789859:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878449.789862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878449.789870:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878449.789871:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511500800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878449.789885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878449.789904:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878449/real 1788878449] req@ffffa06188870700 x1875771511500544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878449.789914:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511500544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878449.789920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878449.789922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878449.789924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878449.789947:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878449.789962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878449.789969:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878449/real 1788878449] req@ffffa06188872680 x1875771511500672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878449.789978:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511500672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878449.789983:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878449.789985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878449.789987:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878449.789991:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878449.790000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878449.790006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878449/real 1788878449] req@ffffa06188872d80 x1875771511500800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878449.790018:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511500800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878449.790026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878449.790029:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878449.790033:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511500800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878450.429464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878450.429562:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878450.685470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878451.453502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878451.453600:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878451.709459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878452.477475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878452.477566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878452.733535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878453.501569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878453.501696:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878453.757603:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878454.525470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878454.525556:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878454.781485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878454.909490:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878454.909504:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878454.909509:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878454.909523:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3441 00000100:00080000:3.0:1788878454.909537:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878454.909557:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878454.909612:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511500928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878454.909641:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878454.909645:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878454.909647:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878454.909651:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3441 00000100:00080000:3.0:1788878454.909655:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878454.909659:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878454.909670:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878454.909681:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511500928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878454.909688:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511501056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878454.909700:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878454.909703:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878454.909704:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878454.909707:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3441 00000100:00080000:3.0:1788878454.909711:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878454.909714:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878454.909719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878454.909730:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511501184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878454.909740:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878454.909744:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878454.909745:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878454.909748:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3441 00000100:00080000:3.0:1788878454.909751:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878454.909753:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878454.909758:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878454.909771:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511501312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878454.909774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878454.909789:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878454.909795:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878454.909798:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878454.909804:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3441 00000100:00080000:3.0:1788878454.909811:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878454.909817:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878454.909823:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878454/real 1788878454] req@ffffa06188871880 x1875771511500928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878454.909828:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878454.909853:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511500928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878454.909857:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511501440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878454.909874:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878454.909882:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878454.909888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511500928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878454.909940:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878454.909964:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878454.909975:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878454/real 1788878454] req@ffffa06188871c00 x1875771511501056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878454.909986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511501056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878454.909994:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878454.909997:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878454.910000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878454.910005:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878454.910018:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878454.910025:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878454/real 1788878454] req@ffffa06188870380 x1875771511501184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878454.910036:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511501184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878454.910043:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878454.910045:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878454.910047:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878454.910052:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878454.910075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878454.910084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878454/real 1788878454] req@ffffa06188870a80 x1875771511501312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878454.910100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511501312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878454.910111:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878454.910115:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878454.910120:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878454.910128:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878454.910152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878454.910163:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878454/real 1788878454] req@ffffa06188872d80 x1875771511501440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878454.910173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511501440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878454.910186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878454.910189:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878454.910192:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878455.549583:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878455.549663:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878455.805539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878456.573502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878456.573628:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878456.829520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878457.597528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878457.597595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878457.853457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878458.621461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878458.621553:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878458.877449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878459.645469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878459.645586:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878459.901503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878460.029542:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878460.029555:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878460.029561:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878460.029575:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3446 00000100:00080000:2.0:1788878460.029589:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878460.029598:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878460.029650:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511501568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878460.029685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878460.029689:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878460.029691:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878460.029695:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3446 00000100:00080000:2.0:1788878460.029698:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878460.029702:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878460.029716:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878460.029733:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511501696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878460.029750:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511501568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878460.029754:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878460.029772:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878460.029774:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878460.029779:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3446 00000100:00080000:2.0:1788878460.029782:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878460.029784:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878460.029788:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878460.029801:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511501824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878460.029810:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878460.029814:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878460.029815:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878460.029818:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3446 00000100:00080000:2.0:1788878460.029821:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878460.029824:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878460.029827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878460.029835:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511501952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878460.029842:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878460.029845:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878460.029847:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878460.029850:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3446 00000100:00080000:2.0:1788878460.029853:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878460.029875:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878460.029880:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878460.029908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878460.029922:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511502080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878460.029983:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878460/real 1788878460] req@ffffa061888aca80 x1875771511501568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878460.030007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511501568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878460.030019:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878460.030023:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878460.030027:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511501568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878460.030055:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878460.030081:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878460.030094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878460/real 1788878460] req@ffffa061888adc00 x1875771511501696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878460.030105:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511501696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878460.030114:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878460.030127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878460.030131:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878460.030136:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878460.030152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878460.030161:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878460/real 1788878460] req@ffffa061888ae300 x1875771511501824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878460.030171:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511501824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878460.030179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878460.030181:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878460.030184:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878460.030190:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878460.030205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878460.030214:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878460/real 1788878460] req@ffffa061888af480 x1875771511501952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878460.030224:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511501952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878460.030232:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878460.030235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878460.030238:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511501952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878460.030244:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878460.030259:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878460.030267:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878460/real 1788878460] req@ffffa061888ac000 x1875771511502080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878460.030277:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511502080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878460.030285:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878460.030288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878460.030290:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878460.669476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878460.669500:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878460.925497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878461.693476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878461.693560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878461.949487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878462.717464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878462.717578:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878462.973471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878463.741589:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878463.741650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878463.997463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878464.765492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878464.765635:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878465.021484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878465.149569:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878465.149585:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878465.149592:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878465.149608:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3451 00000100:00080000:2.0:1788878465.149626:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878465.149637:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878465.149698:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511502208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878465.149739:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878465.149743:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878465.149746:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878465.149751:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3451 00000100:00080000:2.0:1788878465.149755:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878465.149759:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878465.149773:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878465.149802:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511502336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878465.149819:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878465.149845:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511502208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878465.149846:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878465.149852:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878465.149875:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3451 00000100:00080000:2.0:1788878465.149888:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878465.149900:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878465.149909:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878465.149940:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511502464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878465.149952:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878465.149956:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878465.149958:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878465.149963:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3451 00000100:00080000:2.0:1788878465.149969:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878465.149973:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878465.149979:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878465.149993:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511502592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878465.150005:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878465.150009:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878465.150011:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878465.150014:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3451 00000100:00080000:2.0:1788878465.150018:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878465.150033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878465.150038:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878465.150046:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878465.150063:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511502720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878465.150134:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878465/real 1788878465] req@ffffa061888ad500 x1875771511502208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878465.150167:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511502208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878465.150183:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878465.150188:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878465.150194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511502208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878465.150243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878465.150295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878465.150322:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878465/real 1788878465] req@ffffa061888adf80 x1875771511502336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878465.150349:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511502336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878465.150362:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878465.150366:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878465.150371:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878465.150415:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878465.150464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878465.150480:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878465/real 1788878465] req@ffffa061888ac000 x1875771511502464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878465.150510:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511502464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878465.150524:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878465.150531:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878465.150537:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878465.150551:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878465.150592:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878465.150611:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878465/real 1788878465] req@ffffa061888af480 x1875771511502592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878465.150626:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511502592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878465.150636:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878465.150642:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878465.150659:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878465.150672:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878465.150709:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878465.150748:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878465/real 1788878465] req@ffffa061888ae300 x1875771511502720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878465.150773:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511502720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878465.150792:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878465.150796:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878465.150800:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878465.789538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878465.789649:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878466.045535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878466.813523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878466.813599:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878467.069582:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878467.837490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878467.837588:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878468.093521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878468.861465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878468.861660:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878469.117465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878469.885526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878469.885695:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878470.141552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878470.269605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878470.269621:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878470.269628:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878470.269647:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3456 00000100:00080000:2.0:1788878470.269667:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878470.269679:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878470.269760:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511502848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878470.269827:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878470.269833:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878470.269835:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878470.269841:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3456 00000100:00080000:2.0:1788878470.269847:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878470.269852:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878470.269867:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878470.269887:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511502976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878470.269915:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878470.269920:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878470.269922:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878470.269931:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3456 00000100:00100000:0.0:1788878470.269934:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511502848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878470.269936:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878470.269949:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878470.269959:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878470.269978:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511503104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878470.269991:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878470.269996:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878470.269999:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878470.270004:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3456 00000100:00080000:2.0:1788878470.270009:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878470.270014:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878470.270020:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878470.270033:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511503232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878470.270045:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878470.270050:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878470.270056:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878470.270058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878470.270065:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3456 00000100:00080000:2.0:1788878470.270071:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878470.270076:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878470.270083:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878470.270100:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511503360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878470.270125:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878470/real 1788878470] req@ffffa061888adc00 x1875771511502848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878470.270154:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511502848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878470.270170:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878470.270176:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878470.270182:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511502848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878470.270217:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878470.270246:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878470.270263:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878470/real 1788878470] req@ffffa061888aca80 x1875771511502976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878470.270283:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511502976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878470.270301:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878470.270308:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878470.270315:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511502976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878470.270328:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878470.270370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878470.270432:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878470/real 1788878470] req@ffffa061888afb80 x1875771511503104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878470.270450:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511503104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878470.270461:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878470.270466:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878470.270470:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878470.270478:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878470.270507:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878470.270521:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878470/real 1788878470] req@ffffa061888aea00 x1875771511503232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878470.270536:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511503232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878470.270547:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878470.270567:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878470.270571:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878470.270579:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878470.270602:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878470.270623:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878470/real 1788878470] req@ffffa061888af100 x1875771511503360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878470.270647:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511503360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878470.270664:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878470.270671:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878470.270678:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503360:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788878470.909653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878470.909664:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788878471.165502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878471.933562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878471.933656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878472.189512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878472.957515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878472.957714:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878473.213503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878473.981495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878473.981705:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878474.237578:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878475.005524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878475.005638:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878475.261508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878475.389548:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878475.389567:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878475.389575:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878475.389593:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3462 00000100:00080000:2.0:1788878475.389609:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878475.389623:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878475.389679:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511503488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878475.389732:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878475.389741:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878475.389745:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878475.389789:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3462 00000100:00080000:2.0:1788878475.389800:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878475.389810:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878475.389818:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511503488:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878475.389842:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878475.389879:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511503616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878475.389906:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878475.389913:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878475.389917:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878475.389923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878475.389932:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3462 00000100:00080000:2.0:1788878475.389945:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878475.389954:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878475.389966:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878475.389977:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878475/real 1788878475] req@ffffa061888ae680 x1875771511503488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878475.390007:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511503744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878475.390015:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511503488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878475.390069:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878475.390083:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878475.390085:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788878475.390098:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788878475.390104:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511503488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878475.390106:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878475.390125:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3462 00000100:00080000:2.0:1788878475.390135:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878475.390143:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878475.390155:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878475.390157:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878475.390193:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511503872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878475.390211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878475.390227:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878475.390236:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788878475.390239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878475/real 1788878475] req@ffffa061888ac380 x1875771511503616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878475.390243:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878475.390264:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3462 00000100:00100000:0.0:1788878475.390274:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511503616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878475.390275:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878475.390295:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878475.390296:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878475.390310:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000008:00100000:2.0:1788878475.390313:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878475.390316:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878475.390333:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878475.390353:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511504000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878475.390367:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878475.390419:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878475/real 1788878475] req@ffffa061888ad880 x1875771511503744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878475.390450:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511503744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878475.390463:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878475.390469:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878475.390474:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878475.390499:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878475.390523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878475.390537:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878475/real 1788878475] req@ffffa061888aed80 x1875771511503872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878475.390551:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511503872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878475.390561:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878475.390565:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878475.390569:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511503872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878475.390576:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878475.390593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878475.390604:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878475/real 1788878475] req@ffffa061888ac700 x1875771511504000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878475.390617:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511504000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878475.390635:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878475.390639:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878475.390643:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878476.029476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878476.029567:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878476.285508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878477.053538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878477.053683:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878477.309496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878478.077500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878478.077720:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878478.333482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878479.101496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878479.101623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878479.357521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878480.125535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878480.125598:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878480.381484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878480.509530:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878480.509543:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878480.509548:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878480.509561:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3467 00000100:00080000:2.0:1788878480.509576:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878480.509596:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878480.509637:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511504128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878480.509675:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878480.509681:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878480.509685:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878480.509693:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3467 00000100:00080000:2.0:1788878480.509701:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878480.509709:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878480.509739:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878480.509759:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511504128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878480.509790:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511504256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878480.509818:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878480.509824:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878480.509827:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878480.509834:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3467 00000100:00080000:2.0:1788878480.509842:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878480.509849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878480.509858:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878480.509870:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878480.509908:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511504384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878480.509919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878480/real 1788878480] req@ffffa061888ad180 x1875771511504128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878480.509926:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878480.509939:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878480.509951:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878480.509960:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3467 00000100:00080000:2.0:1788878480.509965:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878480.509971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511504128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878480.509975:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878480.509986:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878480.510017:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788878480.510026:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511504512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878480.510031:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878480.510046:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511504128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878480.510063:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878480.510069:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878480.510083:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878480.510090:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3467 00000100:00080000:2.0:1788878480.510096:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878480.510097:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504256:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878480.510106:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878480.510114:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878480.510130:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511504640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878480.510164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878480.510184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878480/real 1788878480] req@ffffa061888ace00 x1875771511504256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878480.510198:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511504256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878480.510208:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878480.510211:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878480.510215:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878480.510221:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878480.510239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878480.510248:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878480/real 1788878480] req@ffffa061888ac700 x1875771511504384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878480.510260:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511504384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878480.510267:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878480.510270:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878480.510273:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878480.510279:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878480.510304:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878480.510315:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878480/real 1788878480] req@ffffa061888aed80 x1875771511504512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878480.510327:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511504512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878480.510347:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878480.510350:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878480.510354:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878480.510369:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878480.510414:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878480.510426:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878480/real 1788878480] req@ffffa061888ad880 x1875771511504640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878480.510438:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511504640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878480.510447:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878480.510450:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878480.510453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878481.149481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878481.149579:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878481.405517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878482.173472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878482.173532:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878482.429508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878483.197474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878483.197537:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878483.453528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878484.221524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878484.221528:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878484.477556:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878485.245559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878485.245645:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878485.501582:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878485.629624:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878485.629647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878485.629657:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878485.629678:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3472 00000100:00080000:0.0:1788878485.629740:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878485.629753:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878485.629818:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511504768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878485.629872:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878485.629877:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878485.629879:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878485.629885:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3472 00000100:00080000:0.0:1788878485.629890:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878485.629895:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878485.629909:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878485.629927:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511504896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878485.629941:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878485.629946:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878485.629949:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878485.629954:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3472 00000100:00080000:0.0:1788878485.629961:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878485.629964:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511504768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878485.629971:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878485.629985:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878485.629999:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511505024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878485.630020:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878485.630025:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878485.630027:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878485.630032:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3472 00000100:00080000:0.0:1788878485.630035:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878485.630040:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878485.630047:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878485.630059:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511505152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878485.630070:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878485.630075:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878485.630077:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788878485.630085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878485.630103:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3472 00000100:00080000:0.0:1788878485.630110:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878485.630116:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878485.630122:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878485.630141:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511505280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878485.630152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878485/real 1788878485] req@ffffa06198afe300 x1875771511504768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878485.630182:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511504768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878485.630202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878485.630210:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878485.630219:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511504768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878485.630234:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878485.630281:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878485.630301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878485/real 1788878485] req@ffffa06198afc380 x1875771511504896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878485.630316:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511504896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878485.630327:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878485.630331:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878485.630335:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511504896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878485.630369:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878485.630442:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878485.630457:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878485/real 1788878485] req@ffffa06198afca80 x1875771511505024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878485.630471:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511505024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878485.630490:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878485.630494:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878485.630498:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878485.630505:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878485.630526:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878485.630537:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878485/real 1788878485] req@ffffa06198afd500 x1875771511505152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878485.630550:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511505152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878485.630560:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878485.630563:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878485.630566:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878485.630573:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878485.630591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878485.630602:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878485/real 1788878485] req@ffffa06198afd180 x1875771511505280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878485.630615:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511505280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878485.630624:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878485.630627:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878485.630632:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878486.269535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878486.269626:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878486.525483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878487.293442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878487.293487:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878487.549436:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878488.317469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878488.317537:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878488.573446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878489.341534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878489.341639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878489.597523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878490.365508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878490.365714:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878490.621472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878490.749487:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878490.749495:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878490.749498:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878490.749506:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3477 00000100:00080000:0.0:1788878490.749515:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878490.749522:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878490.749561:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511505408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878490.749581:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878490.749584:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878490.749585:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878490.749588:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3477 00000100:00080000:0.0:1788878490.749591:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878490.749593:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878490.749601:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878490.749610:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511505536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878490.749617:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878490.749620:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878490.749621:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878490.749623:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3477 00000100:00080000:0.0:1788878490.749625:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878490.749627:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878490.749631:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878490.749637:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511505664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878490.749644:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878490.749659:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878490.749660:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878490.749664:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3477 00000100:00080000:0.0:1788878490.749666:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878490.749669:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878490.749674:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878490.749682:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511505408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878490.749684:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511505792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878490.749697:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878490.749704:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878490.749705:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878490.749708:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3477 00000100:00080000:0.0:1788878490.749710:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878490.749712:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878490.749715:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878490.749726:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511505920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878490.749750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878490.749779:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878490/real 1788878490] req@ffffa06198afd180 x1875771511505408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878490.749793:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511505408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878490.749801:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878490.749804:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878490.749807:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511505408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878490.749812:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878490.749826:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878490.749833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878490/real 1788878490] req@ffffa06198afd500 x1875771511505536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878490.749841:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511505536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878490.749846:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878490.749848:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878490.749850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878490.749853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878490.749864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878490.749871:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878490/real 1788878490] req@ffffa06198afca80 x1875771511505664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878490.749878:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511505664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878490.749883:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878490.749885:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878490.749887:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878490.749907:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878490.749918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878490.749924:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878490/real 1788878490] req@ffffa06198afc380 x1875771511505792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878490.749942:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511505792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878490.749947:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878490.749949:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878490.749950:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878490.749954:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878490.749963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878490.749969:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878490/real 1788878490] req@ffffa06198afe300 x1875771511505920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878490.749976:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511505920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878490.749981:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878490.749984:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878490.749986:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511505920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878491.389455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878491.389532:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878491.645478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878492.413498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878492.413702:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878492.669526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878493.437456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878493.437564:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878493.693535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878494.461577:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878494.461787:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878494.717497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878495.485461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878495.485543:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878495.741523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878495.869557:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878495.869569:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878495.869574:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878495.869588:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3482 00000100:00080000:0.0:1788878495.869602:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878495.869657:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878495.869721:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511506048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878495.869758:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878495.869763:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878495.869766:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878495.869771:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3482 00000100:00080000:0.0:1788878495.869776:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878495.869781:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878495.869793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878495.869827:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511506176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878495.869842:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878495.869846:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878495.869848:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878495.869857:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3482 00000100:00080000:0.0:1788878495.869864:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878495.869867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511506048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878495.869872:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878495.869886:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878495.869902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511506304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878495.869915:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878495.869919:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878495.869922:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878495.869926:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3482 00000100:00080000:0.0:1788878495.869929:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878495.869933:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878495.869938:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878495.869953:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511506432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878495.869965:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878495.869978:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878495.869980:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878495.869984:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3482 00000100:00080000:0.0:1788878495.869999:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878495.870003:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878495.870008:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878495.870017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878495.870030:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511506560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878495.870097:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878495/real 1788878495] req@ffffa06198afea00 x1875771511506048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878495.870162:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511506048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878495.870180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878495.870187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878495.870206:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511506048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878495.870223:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878495.870277:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878495.870318:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878495/real 1788878495] req@ffffa06198afc000 x1875771511506176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878495.870341:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511506176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878495.870352:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878495.870356:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878495.870360:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878495.870391:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878495.870501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878495.870520:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878495/real 1788878495] req@ffffa06198afc700 x1875771511506304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878495.870536:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511506304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878495.870546:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878495.870550:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878495.870554:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878495.870572:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878495.870655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878495.870672:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878495/real 1788878495] req@ffffa06198afdc00 x1875771511506432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878495.870686:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511506432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878495.870725:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878495.870728:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878495.870732:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878495.870740:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878495.870766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878495.870777:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878495/real 1788878495] req@ffffa06198afdf80 x1875771511506560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878495.870792:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511506560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878495.870829:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878495.870833:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878495.870836:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878496.509478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878496.509594:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878496.765568:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878497.533500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878497.533582:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878497.789564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878498.557519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878498.557686:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878498.813505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878499.581520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878499.581664:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878499.837501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878500.605485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878500.605668:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878500.861522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878500.989590:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878500.989603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878500.989608:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878500.989620:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3487 00000100:00080000:1.0:1788878500.989633:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878500.989643:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878500.989685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511506688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878500.989745:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878500.989754:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878500.989759:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878500.989770:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3487 00000100:00080000:1.0:1788878500.989780:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878500.989790:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878500.989826:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878500.989828:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511506688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878500.989858:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511506816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878500.989873:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878500.989876:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878500.989879:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878500.989884:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3487 00000100:00080000:1.0:1788878500.989899:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878500.989903:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878500.989910:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878500.989916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878500.989929:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511506944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878500.989941:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878500.989945:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878500.989947:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878500.989950:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3487 00000100:00080000:1.0:1788878500.989954:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878500.989958:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878500.989963:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878500.989972:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878500/real 1788878500] req@ffffa0618869aa00 x1875771511506688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878500.989977:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511507072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878500.990018:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878500.990022:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878500.990024:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878500.990031:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3487 00000100:00100000:0.0:1788878500.990038:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511506688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878500.990040:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878500.990054:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878500.990067:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878500.990071:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878500.990079:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878500.990085:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511506688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878500.990093:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511507200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878500.990118:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878500.990149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878500.990162:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878500/real 1788878500] req@ffffa0618869b800 x1875771511506816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878500.990175:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511506816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878500.990183:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878500.990187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878500.990190:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878500.990196:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878500.990212:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878500.990221:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878500/real 1788878500] req@ffffa06188699c00 x1875771511506944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878500.990231:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511506944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878500.990238:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878500.990241:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878500.990244:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511506944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878500.990249:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878500.990263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878500.990271:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878500/real 1788878500] req@ffffa06188698380 x1875771511507072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878500.990281:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511507072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878500.990288:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878500.990291:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878500.990294:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878500.990299:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878500.990314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878500.990322:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878500/real 1788878500] req@ffffa0618869b100 x1875771511507200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878500.990332:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511507200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878500.990340:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878500.990342:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878500.990345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878501.629484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878501.629566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878501.885486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878502.653474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878502.653697:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878502.909620:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878503.677480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878503.677654:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878503.933614:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878504.701585:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878504.701741:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878504.957632:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878505.725501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878505.725722:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878505.981499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878506.109524:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878506.109533:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878506.109537:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878506.109561:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3492 00000100:00080000:3.0:1788878506.109573:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878506.109583:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878506.109623:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511507328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878506.109647:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878506.109650:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878506.109651:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878506.109655:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3492 00000100:00080000:3.0:1788878506.109658:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878506.109662:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878506.109670:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878506.109681:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511507456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878506.109697:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511507328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878506.109697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878506.109703:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878506.109707:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878506.109713:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3492 00000100:00080000:3.0:1788878506.109715:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878506.109718:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878506.109721:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878506.109730:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511507584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878506.109737:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878506.109740:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878506.109741:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878506.109743:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3492 00000100:00080000:3.0:1788878506.109745:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878506.109748:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878506.109751:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878506.109758:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511507712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878506.109764:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878506.109767:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878506.109768:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878506.109770:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3492 00000100:00080000:3.0:1788878506.109773:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878506.109776:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878506.109780:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878506.109787:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511507840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878506.109821:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878506.109860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878506.109893:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878506.109924:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878506.749586:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878506.749695:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878507.005518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878507.773599:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878507.773698:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878508.029489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878508.797476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878508.797536:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878509.053560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878509.181655:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878509.181692:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878509.181707:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878509.181752:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878509.181758:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878509.181766:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878509.181777:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878509.181798:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878506/real 1788878509] req@ffffa06188870700 x1875771511507456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878509.181858:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511507456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878509.181878:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878509.181896:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878509.181903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878509.181929:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878506/real 1788878509] req@ffffa06188873b80 x1875771511507584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878509.181951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511507584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878509.181985:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878509.181989:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878509.181997:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878509.182005:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878506/real 1788878509] req@ffffa06188872300 x1875771511507712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878509.182019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511507712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878509.182028:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878509.182032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878509.182036:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878509.182047:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878506/real 1788878509] req@ffffa06188873800 x1875771511507840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878509.182060:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511507840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878509.182069:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878509.182074:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878509.182078:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511507840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878509.182085:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878506/real 1788878509] req@ffffa06188872680 x1875771511507328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878509.182099:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511507328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878509.182108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878509.182112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878509.182116:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511507328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878509.821511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878509.821625:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878510.077458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878510.845485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878510.845564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878511.101460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878511.229498:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878511.229519:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878511.229524:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878511.229535:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3497 00000100:00080000:2.0:1788878511.229571:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878511.229584:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878511.229665:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511507968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878511.229709:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878511.229715:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878511.229719:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878511.229741:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3497 00000100:00080000:2.0:1788878511.229746:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878511.229754:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878511.229763:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511507968:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878511.229771:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878511.229796:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511508096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878511.229813:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878511.229818:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878511.229821:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878511.229826:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3497 00000100:00080000:2.0:1788878511.229830:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878511.229835:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878511.229845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878511.229845:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878511.229867:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511508224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878511.229879:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878511.229884:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878511.229886:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878511.229896:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3497 00000100:00100000:0.0:1788878511.229906:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878511/real 1788878511] req@ffffa061888ac380 x1875771511507968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878511.229908:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878511.229919:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878511.229934:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878511.229950:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511507968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878511.229954:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511508352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878511.230003:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878511.230006:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878511.230010:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878511.230015:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878511.230018:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788878511.230025:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3497 00000100:00100000:0.0:1788878511.230033:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511507968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878511.230037:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878511.230049:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878511.230056:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878511.230073:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878511.230076:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511508480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878511.230115:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878511.230137:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878511/real 1788878511] req@ffffa061888ae680 x1875771511508096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878511.230150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511508096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878511.230159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878511.230163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878511.230166:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878511.230174:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878511.230196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878511.230207:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878511/real 1788878511] req@ffffa061888af100 x1875771511508224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878511.230219:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511508224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878511.230227:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878511.230231:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878511.230234:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878511.230243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878511.230265:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878511.230277:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878511/real 1788878511] req@ffffa061888aea00 x1875771511508352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878511.230290:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511508352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878511.230299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878511.230303:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878511.230306:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878511.230329:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878511.230351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878511.230362:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878511/real 1788878511] req@ffffa061888afb80 x1875771511508480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878511.230377:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511508480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878511.230423:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878511.230430:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878511.230435:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878511.869466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878511.869614:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878512.125513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878512.893444:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878512.893559:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878513.149565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878513.917537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878513.917708:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878514.173548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878514.941657:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878514.941713:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878515.197469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878515.965551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878515.965615:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878516.221507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878516.349590:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878516.349601:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878516.349606:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878516.349615:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3503 00000100:00080000:1.0:1788878516.349627:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878516.349635:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878516.349675:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511508608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878516.349702:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878516.349705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878516.349707:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878516.349711:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3503 00000100:00080000:1.0:1788878516.349714:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878516.349719:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878516.349729:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878516.349743:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511508608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878516.349744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511508736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878516.349760:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878516.349763:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878516.349764:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878516.349768:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3503 00000100:00080000:1.0:1788878516.349771:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878516.349774:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878516.349778:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878516.349789:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511508864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878516.349797:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878516.349801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878516.349802:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878516.349805:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3503 00000100:00080000:1.0:1788878516.349809:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878516.349815:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878516.349816:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878516.349823:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878516.349834:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511508992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878516.349843:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878516.349845:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878516.349847:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878516.349851:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3503 00000100:00080000:1.0:1788878516.349855:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878516.349858:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878516.349862:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878516.349869:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878516/real 1788878516] req@ffffa0618869b100 x1875771511508608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878516.349875:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511509120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878516.349925:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511508608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878516.349942:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878516.349947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878516.349951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511508608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878516.349975:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878516.350000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878516.350012:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878516/real 1788878516] req@ffffa06188698380 x1875771511508736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878516.350029:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511508736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878516.350040:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878516.350045:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878516.350049:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878516.350057:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878516.350088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878516.350103:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878516/real 1788878516] req@ffffa06188699c00 x1875771511508864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878516.350114:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511508864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878516.350121:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878516.350124:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878516.350127:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878516.350132:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878516.350149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878516.350157:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878516/real 1788878516] req@ffffa0618869b800 x1875771511508992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878516.350166:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511508992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878516.350173:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878516.350175:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878516.350177:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511508992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878516.350182:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878516.350194:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878516.350201:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878516/real 1788878516] req@ffffa0618869aa00 x1875771511509120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878516.350217:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511509120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878516.350223:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878516.350225:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878516.350228:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878516.989481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878516.989617:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878517.245520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878518.013582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878518.013771:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878518.269560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878519.037532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878519.037606:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878519.293490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878520.061578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878520.061736:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878520.317553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878521.085466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878521.085551:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878521.341485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878521.469527:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878521.469536:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878521.469541:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878521.469553:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3508 00000100:00080000:1.0:1788878521.469565:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878521.469574:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878521.469625:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511509248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878521.469651:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878521.469654:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878521.469656:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878521.469660:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3508 00000100:00080000:1.0:1788878521.469664:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878521.469668:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878521.469679:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878521.469687:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511509248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878521.469697:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511509376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878521.469711:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878521.469715:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878521.469717:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878521.469721:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3508 00000100:00080000:1.0:1788878521.469726:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878521.469731:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878521.469737:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878521.469747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511509504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878521.469760:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878521.469763:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878521.469767:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878521.469769:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878521.469785:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3508 00000100:00080000:1.0:1788878521.469789:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878521.469792:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878521.469796:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878521.469806:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511509632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878521.469830:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878521.469831:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878521/real 1788878521] req@ffffa0618869a680 x1875771511509248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878521.469836:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878521.469840:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878521.469849:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3508 00000100:00080000:1.0:1788878521.469857:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878521.469861:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878521.469866:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878521.469873:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511509248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878521.469877:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511509760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878521.469918:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878521.469925:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878521.469931:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511509248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878521.469985:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878521.470007:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878521.470020:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878521/real 1788878521] req@ffffa0618869bb80 x1875771511509376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878521.470048:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511509376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878521.470055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878521.470059:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878521.470063:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878521.470069:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878521.470107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878521.470120:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878521/real 1788878521] req@ffffa0618869ad80 x1875771511509504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878521.470134:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511509504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878521.470140:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878521.470142:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878521.470145:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878521.470149:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878521.470176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878521.470184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878521/real 1788878521] req@ffffa06188698000 x1875771511509632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878521.470192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511509632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878521.470199:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878521.470201:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878521.470203:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878521.470207:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878521.470217:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878521.470223:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878521/real 1788878521] req@ffffa06188699f80 x1875771511509760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878521.470230:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511509760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878521.470236:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878521.470238:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878521.470240:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511509760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878522.109538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878522.109625:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878522.365518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878523.133514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878523.133661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878523.389511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878524.157545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878524.157697:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878524.413618:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878525.181571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878525.181653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878525.437619:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878526.205484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878526.205637:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878526.461562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878526.589499:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878526.589507:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878526.589512:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878526.589522:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3513 00000100:00080000:1.0:1788878526.589531:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878526.589537:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878526.589571:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511509888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878526.589593:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878526.589595:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878526.589597:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878526.589600:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3513 00000100:00080000:1.0:1788878526.589602:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878526.589605:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878526.589613:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878526.589622:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511510016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878526.589626:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511509888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878526.589638:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878526.589641:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878526.589642:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878526.589645:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3513 00000100:00080000:1.0:1788878526.589647:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878526.589649:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878526.589653:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878526.589660:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511510144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878526.589671:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878526.589674:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878526.589675:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878526.589677:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3513 00000100:00080000:1.0:1788878526.589681:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878526.589683:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878526.589686:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878526.589690:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878526.589695:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511510272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878526.589702:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878526.589705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878526.589707:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878526.589710:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3513 00000100:00080000:1.0:1788878526.589712:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878526.589715:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878526.589719:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878526.589724:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878526/real 1788878526] req@ffffa06188699f80 x1875771511509888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878526.589728:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511510400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878526.589764:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511509888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878526.589773:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878526.589776:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878526.589778:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511509888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878526.589799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878526.589822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878526.589836:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878526/real 1788878526] req@ffffa06188698000 x1875771511510016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878526.589850:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511510016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878526.589857:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878526.589859:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878526.589862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878526.589866:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878526.589881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878526.589889:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878526/real 1788878526] req@ffffa0618869ad80 x1875771511510144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878526.589897:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511510144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878526.589906:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878526.589908:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878526.589910:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878526.589914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878526.589924:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878526.589930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878526/real 1788878526] req@ffffa0618869bb80 x1875771511510272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878526.589937:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511510272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878526.589943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878526.589944:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878526.589946:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878526.589950:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878526.589960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878526.589966:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878526/real 1788878526] req@ffffa0618869a680 x1875771511510400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878526.589973:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511510400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878526.589977:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878526.589979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878526.589981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878527.229501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878527.229598:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878527.485614:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878528.253537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878528.253673:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878528.509535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878529.277584:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878529.277637:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878529.533602:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878530.301535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878530.301668:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878530.557571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878531.325538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878531.325655:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878531.581588:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878531.709600:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878531.709616:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878531.709623:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878531.709641:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3518 00000100:00080000:2.0:1788878531.709660:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878531.709673:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878531.709746:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511510528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878531.709802:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878531.709808:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878531.709811:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878531.709819:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3518 00000100:00080000:2.0:1788878531.709829:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878531.709841:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878531.709863:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511510528:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878531.709863:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878531.709905:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511510656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878531.709942:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878531.709949:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878531.709954:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878531.709980:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3518 00000100:00080000:2.0:1788878531.709988:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878531.709996:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878531.710006:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878531.710017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878531.710038:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511510784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878531.710058:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878531.710062:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878531.710065:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878531.710072:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3518 00000100:00080000:2.0:1788878531.710078:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878531.710089:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878531.710090:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878531/real 1788878531] req@ffffa061888aca80 x1875771511510528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878531.710097:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878531.710120:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511510912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878531.710131:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511510528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878531.710138:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878531.710150:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878531.710155:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878531.710164:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3518 00000100:00080000:0.0:1788878531.710166:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878531.710169:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878531.710181:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788878531.710182:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878531.710194:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878531.710199:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511510528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878531.710212:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511511040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878531.710261:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878531.710299:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878531.710315:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878531/real 1788878531] req@ffffa061888adc00 x1875771511510656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878531.710331:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511510656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878531.710343:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878531.710348:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878531.710352:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878531.710360:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878531.710462:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878531.710502:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878531/real 1788878531] req@ffffa061888ae300 x1875771511510784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878531.710518:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511510784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878531.710529:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878531.710534:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878531.710538:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878531.710546:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878531.710598:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878531.710608:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878531/real 1788878531] req@ffffa061888af480 x1875771511510912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878531.710621:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511510912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878531.710630:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878531.710632:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878531.710653:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511510912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878531.710661:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878531.710683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878531.710695:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878531/real 1788878531] req@ffffa061888ac000 x1875771511511040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878531.710709:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511511040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878531.710719:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878531.710722:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878531.710726:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878532.349525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878532.349680:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878532.605491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878533.373538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878533.373801:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878533.629513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878534.397529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878534.397624:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878534.653552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878535.421532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878535.421617:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878535.677466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878536.445520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878536.445587:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878536.701526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878536.829560:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878536.829581:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878536.829592:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878536.829613:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3523 00000100:00080000:2.0:1788878536.829638:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878536.829652:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878536.829732:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511511168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878536.829774:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878536.829779:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878536.829782:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878536.829788:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3523 00000100:00080000:2.0:1788878536.829793:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878536.829798:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878536.829814:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878536.829833:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511511296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878536.829852:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878536.829864:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788878536.829867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511511168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878536.829869:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878536.829886:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3523 00000100:00080000:2.0:1788878536.829897:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878536.829903:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878536.829910:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878536.829931:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511511424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878536.829944:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878536.829949:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878536.829952:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878536.829957:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3523 00000100:00080000:2.0:1788878536.829962:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878536.829966:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878536.829972:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878536.829996:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511511552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878536.830008:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878536.830012:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878536.830014:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878536.830018:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3523 00000100:00080000:2.0:1788878536.830023:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878536.830032:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878536.830049:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878536.830058:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878536.830083:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511511680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878536.830095:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878536/real 1788878536] req@ffffa061888adf80 x1875771511511168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878536.830150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511511168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878536.830174:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878536.830183:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878536.830192:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511511168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878536.830243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878536.830287:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878536.830305:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878536/real 1788878536] req@ffffa061888ad500 x1875771511511296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878536.830322:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511511296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878536.830383:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878536.830390:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878536.830396:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878536.830449:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878536.830480:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878536.830495:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878536/real 1788878536] req@ffffa061888ac000 x1875771511511424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878536.830509:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511511424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878536.830518:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878536.830522:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878536.830526:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878536.830533:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878536.830552:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878536.830562:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878536/real 1788878536] req@ffffa061888af480 x1875771511511552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878536.830575:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511511552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878536.830584:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878536.830587:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878536.830590:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878536.830596:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878536.830622:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878536.830640:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878536/real 1788878536] req@ffffa061888ae300 x1875771511511680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878536.830662:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511511680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878536.830677:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878536.830700:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878536.830707:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878537.469566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878537.469661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878537.725613:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878538.493521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878538.493681:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878538.749491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878539.517526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878539.517672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878539.773527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878540.541475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878540.541579:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878540.797503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878541.565475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878541.565612:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878541.821515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878541.949497:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878541.949510:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878541.949516:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878541.949530:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3528 00000100:00080000:2.0:1788878541.949545:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878541.949554:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878541.949598:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511511808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878541.949632:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878541.949637:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878541.949639:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878541.949644:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3528 00000100:00080000:2.0:1788878541.949648:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878541.949652:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878541.949662:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878541.949677:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511511936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878541.949691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878541.949695:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878541.949697:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878541.949703:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511511808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878541.949705:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3528 00000100:00080000:2.0:1788878541.949713:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878541.949722:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878541.949731:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878541.949745:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511512064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878541.949757:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878541.949761:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878541.949763:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878541.949768:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3528 00000100:00080000:2.0:1788878541.949772:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878541.949776:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878541.949781:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878541.949798:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878541.949799:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511512192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878541.949814:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878541.949818:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878541.949821:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878541.949826:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3528 00000100:00080000:2.0:1788878541.949830:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878541.949834:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878541.949841:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878541.949853:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878541/real 1788878541] req@ffffa061888adc00 x1875771511511808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878541.949855:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511512320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878541.949926:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511511808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878541.949963:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878541.949970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878541.949976:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511511808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878541.950011:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878541.950047:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878541.950063:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878541/real 1788878541] req@ffffa061888aca80 x1875771511511936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878541.950077:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511511936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878541.950086:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878541.950090:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878541.950093:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511511936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878541.950100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878541.950119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878541.950136:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878541/real 1788878541] req@ffffa061888afb80 x1875771511512064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878541.950165:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511512064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878541.950179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878541.950184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878541.950190:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878541.950200:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878541.950231:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878541.950250:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878541/real 1788878541] req@ffffa061888aea00 x1875771511512192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878541.950271:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511512192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878541.950284:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878541.950287:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878541.950291:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878541.950320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878541.950370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878541.950390:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878541/real 1788878541] req@ffffa061888af100 x1875771511512320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878541.950467:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511512320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878541.950480:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878541.950483:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878541.950485:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878542.589499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878542.589637:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878542.845510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878543.613492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878543.613631:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878543.869529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878544.637462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878544.637516:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878544.893458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878545.661511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878545.661591:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878545.917491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878546.685488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878546.685568:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878546.941453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878547.069488:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878547.069497:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878547.069501:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878547.069509:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3533 00000100:00080000:3.0:1788878547.069519:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878547.069525:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878547.069563:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511512448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878547.069588:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878547.069602:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878547.069604:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878547.069609:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3533 00000100:00080000:3.0:1788878547.069613:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878547.069620:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878547.069630:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878547.069633:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511512448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878547.069646:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511512576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878547.069658:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878547.069664:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878547.069665:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878547.069668:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3533 00000100:00080000:3.0:1788878547.069672:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878547.069677:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878547.069693:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878547.069705:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511512704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878547.069712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878547.069720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878547.069724:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878547.069726:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878547.069730:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3533 00000100:00080000:3.0:1788878547.069736:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878547.069740:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878547.069746:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878547.069751:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878547/real 1788878547] req@ffffa06188872a00 x1875771511512448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878547.069759:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511512832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878547.069771:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511512448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878547.069772:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878547.069782:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878547.069795:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878547.069798:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878547.069803:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3533 00000100:00080000:0.0:1788878547.069806:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878547.069811:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878547.069812:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511512448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878547.069819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878547.069824:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878547.069836:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878547.069837:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511512960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878547.069863:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878547.069880:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878547/real 1788878547] req@ffffa06188873100 x1875771511512576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878547.069902:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511512576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878547.069910:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878547.069913:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878547.069915:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878547.069920:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878547.069935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878547.069943:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878547/real 1788878547] req@ffffa06188871500 x1875771511512704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878547.069951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511512704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878547.069956:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878547.069958:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878547.069960:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878547.069964:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878547.069974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878547.069980:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878547/real 1788878547] req@ffffa06188873480 x1875771511512832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878547.069987:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511512832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878547.069996:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878547.069998:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878547.070000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878547.070011:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878547.070021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878547.070027:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878547/real 1788878547] req@ffffa06188872a00 x1875771511512960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878547.070034:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511512960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878547.070039:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878547.070041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878547.070043:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511512960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878547.709448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878547.709531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878547.965540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878548.733542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878548.733623:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878548.989484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878549.757450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878549.757591:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878550.013461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878550.781551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878550.781561:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878551.037535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878551.805476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878551.805620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878552.061481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878552.189550:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878552.189563:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878552.189569:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878552.189582:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3538 00000100:00080000:1.0:1788878552.189596:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878552.189607:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878552.189655:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511513088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878552.189693:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878552.189698:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878552.189701:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878552.189706:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3538 00000100:00080000:1.0:1788878552.189712:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878552.189726:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878552.189739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878552.189764:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511513088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878552.189764:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511513216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878552.189798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878552.189803:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878552.189805:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878552.189809:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3538 00000100:00080000:1.0:1788878552.189814:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878552.189819:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878552.189826:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878552.189842:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511513344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878552.189854:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878552.189859:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878552.189861:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878552.189877:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3538 00000800:00000100:0.0:1788878552.189885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878552.189888:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878552.189897:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878552.189905:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878552.189922:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511513472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878552.189934:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878552.189939:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878552/real 1788878552] req@ffffa0618869aa00 x1875771511513088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878552.189941:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878552.189948:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878552.189963:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3538 00000100:00080000:1.0:1788878552.189973:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878552.189986:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878552.190000:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878552.190012:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511513088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878552.190019:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511513600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878552.190061:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878552.190070:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878552.190076:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511513088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878552.190116:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878552.190146:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878552.190163:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878552/real 1788878552] req@ffffa0618869b800 x1875771511513216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878552.190179:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511513216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878552.190190:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878552.190194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878552.190198:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878552.190206:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878552.190276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878552.190300:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878552/real 1788878552] req@ffffa06188699c00 x1875771511513344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878552.190336:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511513344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878552.190367:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878552.190374:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878552.190413:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878552.190424:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878552.190456:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878552.190472:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878552/real 1788878552] req@ffffa06188698380 x1875771511513472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878552.190488:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511513472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878552.190498:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878552.190502:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878552.190506:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878552.190513:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878552.190532:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878552.190543:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878552/real 1788878552] req@ffffa0618869b100 x1875771511513600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878552.190557:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511513600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878552.190566:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878552.190569:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878552.190574:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878552.829508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878552.829642:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878553.085507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878553.853513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878553.853640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878554.109532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878554.877498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878554.877722:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878555.133580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878555.901465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878555.901544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878556.157520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878556.925542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878556.925623:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878557.181496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878557.309604:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878557.309617:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878557.309624:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878557.309639:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3544 00000100:00080000:2.0:1788878557.309656:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878557.309666:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878557.309725:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511513728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878557.309758:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878557.309763:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878557.309765:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878557.309778:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3544 00000100:00080000:2.0:1788878557.309783:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878557.309788:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878557.309803:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878557.309808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511513728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878557.309828:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511513856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878557.309847:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878557.309853:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878557.309857:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878557.309864:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3544 00000100:00080000:2.0:1788878557.309871:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878557.309879:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878557.309887:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878557.309906:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511513984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878557.309920:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878557.309921:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878557.309928:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878557.309933:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878557.309939:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3544 00000100:00080000:2.0:1788878557.309944:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878557.309949:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878557.309984:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878557.310002:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511514112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878557.310015:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878557.310022:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878557/real 1788878557] req@ffffa061888ae680 x1875771511513728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878557.310023:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878557.310030:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878557.310047:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3544 00000100:00080000:2.0:1788878557.310056:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878557.310062:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878557.310068:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878557.310069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511513728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878557.310086:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511514240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878557.310110:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878557.310120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878557.310128:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511513728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878557.310165:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878557.310251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878557.310267:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878557/real 1788878557] req@ffffa061888ac380 x1875771511513856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878557.310280:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511513856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878557.310289:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878557.310293:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878557.310297:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878557.310303:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878557.310322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878557.310331:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878557/real 1788878557] req@ffffa061888ad880 x1875771511513984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878557.310342:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511513984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878557.310350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878557.310352:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878557.310355:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511513984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878557.310360:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878557.310376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878557.310385:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878557/real 1788878557] req@ffffa061888aed80 x1875771511514112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878557.310420:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511514112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878557.310430:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878557.310433:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878557.310437:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878557.310443:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878557.310465:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878557.310476:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878557/real 1788878557] req@ffffa061888ac700 x1875771511514240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878557.310487:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511514240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878557.310495:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878557.310498:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878557.310501:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878557.949528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878557.949639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878558.205521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878558.973538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878558.973646:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878559.229505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878559.997522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878559.997738:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878560.253535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878561.021504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878561.021641:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878561.277513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878562.045463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878562.045616:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878562.301464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878562.429542:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878562.429554:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878562.429560:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878562.429584:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3549 00000100:00080000:2.0:1788878562.429598:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878562.429606:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878562.431110:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511514368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878562.431165:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878562.431169:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878562.431172:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878562.431178:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3549 00000100:00080000:2.0:1788878562.431184:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878562.431190:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878562.431203:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878562.431206:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511514368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878562.431222:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511514496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878562.431232:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878562.431235:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878562.431237:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878562.431240:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3549 00000100:00080000:2.0:1788878562.431243:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878562.431246:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878562.431250:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878562.431260:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511514624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878562.431266:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878562.431269:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878562.431270:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878562.431272:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3549 00000100:00080000:2.0:1788878562.431275:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878562.431277:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878562.431280:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878562.431286:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511514752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878562.431294:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878562.431295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878562.431299:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878562.431302:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878562.431305:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3549 00000100:00080000:2.0:1788878562.431308:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878562.431311:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878562.431313:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878562.431320:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511514880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878562.431352:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878562/real 1788878562] req@ffffa061888ace00 x1875771511514368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878562.431376:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511514368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878562.431409:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878562.431414:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878562.431419:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511514368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878562.431449:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878562.431488:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878562.431503:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878562/real 1788878562] req@ffffa061888ad180 x1875771511514496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878562.431517:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511514496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878562.431527:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878562.431531:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878562.431535:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878562.431543:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878562.431566:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878562.431579:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878562/real 1788878562] req@ffffa061888ac700 x1875771511514624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878562.431602:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511514624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878562.431611:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878562.431615:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878562.431619:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878562.431625:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878562.431680:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878562.431695:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878562/real 1788878562] req@ffffa061888aed80 x1875771511514752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878562.431712:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511514752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878562.431724:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878562.431728:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878562.431733:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878562.431741:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878562.431769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878562.431785:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878562/real 1788878562] req@ffffa061888ad880 x1875771511514880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878562.431801:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511514880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878562.431823:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878562.431827:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878562.431832:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511514880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878563.069573:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878563.069645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878563.325496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878564.093478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878564.093662:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878564.349533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878565.117521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878565.117573:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878565.373488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878566.141500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878566.141608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878566.397499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878567.165437:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878567.165617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878567.421559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878567.549555:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878567.549574:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878567.549584:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878567.549602:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3554 00000100:00080000:2.0:1788878567.549623:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878567.549638:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878567.549706:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511515008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878567.549764:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878567.549772:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878567.549776:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878567.549788:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3554 00000100:00080000:2.0:1788878567.549796:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878567.549808:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878567.549826:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511515008:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878567.549827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878567.549864:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511515136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878567.549890:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878567.549898:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878567.549902:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878567.549910:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3554 00000100:00080000:2.0:1788878567.549919:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878567.549927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878567.549935:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878567.549948:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878567.549974:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511515264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878567.549986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878567/real 1788878567] req@ffffa061888ac380 x1875771511515008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878567.550018:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878567.550028:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878567.550032:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878567.550035:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511515008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878567.550048:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3554 00000100:00080000:2.0:1788878567.550064:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878567.550072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878567.550076:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878567.550086:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:2.0:1788878567.550094:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878567.550096:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511515008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878567.550158:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878567.550159:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511515392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878567.550195:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878567.550204:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788878567.550207:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878567.550211:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878567.550224:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3554 00000100:00080000:2.0:1788878567.550234:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878567.550236:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878567/real 1788878567] req@ffffa061888ae680 x1875771511515136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878567.550248:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878567.550268:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878567.550281:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511515136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878567.550297:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788878567.550304:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511515520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878567.550309:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878567.550330:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878567.550350:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878567.550414:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878567.550431:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878567/real 1788878567] req@ffffa061888af100 x1875771511515264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878567.550445:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511515264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878567.550461:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878567.550465:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878567.550468:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878567.550492:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878567.550518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878567.550531:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878567/real 1788878567] req@ffffa061888aea00 x1875771511515392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878567.550545:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511515392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878567.550554:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878567.550558:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878567.550562:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878567.550570:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878567.550600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878567.550614:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878567/real 1788878567] req@ffffa061888afb80 x1875771511515520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878567.550627:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511515520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878567.550636:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878567.550639:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878567.550644:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878568.189495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878568.189640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878568.445535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878569.213466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878569.213548:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878569.469511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878570.237541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878570.237600:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878570.493507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878571.261478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878571.261556:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878571.517465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878572.285503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878572.285582:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878572.541483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878572.669469:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878572.669480:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878572.669486:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878572.669500:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3559 00000100:00080000:2.0:1788878572.669514:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878572.669524:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878572.669570:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511515648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878572.669595:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878572.669609:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878572.669611:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878572.669614:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3559 00000100:00080000:2.0:1788878572.669619:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878572.669623:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878572.669632:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878572.669638:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511515648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878572.669644:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511515776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878572.669653:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878572.669655:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878572.669656:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878572.669659:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3559 00000100:00080000:2.0:1788878572.669661:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878572.669663:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878572.669667:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878572.669675:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511515904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878572.669681:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878572.669684:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878572.669685:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878572.669687:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3559 00000100:00080000:2.0:1788878572.669690:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878572.669692:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878572.669695:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878572.669703:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511516032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878572.669714:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878572.669717:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878572.669719:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878572.669721:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3559 00000100:00080000:2.0:1788878572.669724:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878572.669727:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878572.669730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878572.669738:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511516160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878572.669744:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878572.669771:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878572.669787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878572.669808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878573.309484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878573.309622:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878573.565477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878574.333505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878574.333744:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878574.589561:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878575.357503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878575.357598:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878575.613511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788878575.741784:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788878575.741830:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788878575.741848:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878575.741917:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878575.741934:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878575.741968:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788878575.741987:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878575.741991:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878572/real 1788878575] req@ffffa061888adc00 x1875771511515776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878575.742043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511515776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878575.742058:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878575.742081:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878575.742087:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878575.742098:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878572/real 1788878575] req@ffffa061888ae300 x1875771511515904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878575.742109:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511515904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878575.742116:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878575.742119:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878575.742121:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511515904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878575.742127:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878572/real 1788878575] req@ffffa061888af480 x1875771511516032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878575.742140:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511516032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878575.742149:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878575.742153:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878575.742157:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878575.742165:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878572/real 1788878575] req@ffffa061888aca80 x1875771511515648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878575.742178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511515648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878575.742186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878575.742190:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878575.742193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511515648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878575.742231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878572/real 1788878575] req@ffffa061888ac000 x1875771511516160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878575.742247:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511516160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878575.742256:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878575.742261:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878575.742264:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878576.381579:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878576.381646:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878576.637475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878577.405504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878577.405586:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878577.661580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878577.789574:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878577.789588:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878577.789594:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878577.789608:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3564 00000100:00080000:0.0:1788878577.789622:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878577.789633:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878577.789680:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511516288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878577.789733:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878577.789738:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878577.789741:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878577.789747:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3564 00000100:00080000:0.0:1788878577.789751:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878577.789757:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878577.789784:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878577.789805:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511516416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878577.789808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511516288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878577.789848:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878577.789852:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878577.789855:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878577.789862:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3564 00000100:00080000:0.0:1788878577.789866:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878577.789870:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878577.789877:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878577.789893:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511516544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878577.789929:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878577.789935:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878577.789943:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788878577.789943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878577.789958:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3564 00000100:00080000:0.0:1788878577.789966:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878577.789972:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878577.789980:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878577.789998:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511516672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878577.790005:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878577/real 1788878577] req@ffffa06198aff800 x1875771511516288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878577.790040:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878577.790087:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878577.790106:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878577.790120:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3564 00000100:00100000:3.0:1788878577.790136:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511516288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878577.790147:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878577.790159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878577.790163:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788878577.790172:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:0.0:1788878577.790178:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878577.790184:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511516288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878577.790206:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511516800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878577.790277:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878577.790328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878577.790357:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878577/real 1788878577] req@ffffa06198affb80 x1875771511516416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878577.790432:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511516416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878577.790446:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878577.790451:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878577.790457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878577.790467:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878577.790504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878577.790525:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878577/real 1788878577] req@ffffa06198afd880 x1875771511516544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878577.790545:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511516544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878577.790566:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878577.790572:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878577.790577:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878577.790588:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878577.790649:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878577.790664:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878577/real 1788878577] req@ffffa06198aff480 x1875771511516672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878577.790680:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511516672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878577.790700:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878577.790705:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878577.790709:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878577.790716:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878577.790749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878577.790763:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878577/real 1788878577] req@ffffa06198afed80 x1875771511516800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878577.790899:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511516800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878577.790910:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878577.790915:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878577.790934:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511516800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878578.429504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878578.429589:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878578.685440:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878579.453461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878579.453568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878579.709564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878580.477460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878580.477562:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878580.733484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878581.501510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878581.501620:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878581.757466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788878582.525543:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788878582.525547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788878582.781565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878582.909537:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878582.909580:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878582.909588:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878582.909606:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3569 00000100:00080000:2.0:1788878582.909625:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878582.909635:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878582.909727:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511516928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878582.909764:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878582.909768:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878582.909770:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878582.909776:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3569 00000100:00080000:2.0:1788878582.909780:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878582.909784:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878582.909796:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878582.909820:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511517056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878582.909837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878582.909843:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878582.909848:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878582.909855:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3569 00000100:00080000:2.0:1788878582.909862:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878582.909871:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878582.909881:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878582.909909:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511517184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878582.909922:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511516928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878582.909936:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878582.909943:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878582.909946:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878582.909954:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3569 00000100:00080000:2.0:1788878582.909960:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878582.909967:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878582.909987:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878582.910045:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511517312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878582.910054:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878582.910058:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878582.910060:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878582.910065:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3569 00000100:00080000:2.0:1788878582.910069:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878582.910073:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878582.910095:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878582.910098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878582.910113:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511517440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878582.910162:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878582/real 1788878582] req@ffffa061888ad500 x1875771511516928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878582.910183:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511516928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878582.910215:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878582.910220:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878582.910225:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511516928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878582.910237:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878582.910305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878582.910322:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878582/real 1788878582] req@ffffa061888adf80 x1875771511517056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878582.910353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511517056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878582.910385:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878582.910392:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878582.910399:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878582.910493:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878582.910531:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878582.910546:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878582/real 1788878582] req@ffffa061888ac000 x1875771511517184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878582.910577:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511517184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878582.910586:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878582.910590:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878582.910594:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878582.910601:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878582.910622:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878582.910632:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878582/real 1788878582] req@ffffa061888aca80 x1875771511517312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878582.910645:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511517312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878582.910654:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878582.910657:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878582.910660:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878582.910666:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878582.910684:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878582.910693:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878582/real 1788878582] req@ffffa061888af480 x1875771511517440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878582.910705:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511517440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878582.910713:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878582.910716:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878582.910735:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878583.549558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878583.549613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878583.805600:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878584.573465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878584.573573:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878584.829475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878585.597497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878585.597579:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878585.853514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878586.621456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878586.621541:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878586.877465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878587.645468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878587.645566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878587.901492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878588.029508:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878588.029518:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878588.029522:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878588.029533:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3574 00000100:00080000:2.0:1788878588.029544:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878588.029553:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878588.029594:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511517568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878588.029622:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878588.029626:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878588.029628:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878588.029631:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3574 00000100:00080000:2.0:1788878588.029634:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878588.029637:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878588.029647:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878588.029658:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511517696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878588.029668:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878588.029671:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878588.029672:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878588.029676:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3574 00000100:00080000:2.0:1788878588.029679:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878588.029682:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878588.029686:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878588.029690:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511517568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878588.029698:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511517824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878588.029708:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878588.029711:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878588.029714:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878588.029717:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3574 00000100:00080000:2.0:1788878588.029720:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878588.029722:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878588.029727:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878588.029736:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511517952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878588.029744:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878588.029746:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878588.029747:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878588.029750:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3574 00000100:00080000:2.0:1788878588.029755:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878588.029758:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878588.029761:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878588.029767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878588.029773:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511518080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878588.029821:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878588/real 1788878588] req@ffffa061888ae300 x1875771511517568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878588.029838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511517568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878588.029854:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878588.029857:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878588.029861:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511517568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878588.029867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878588.029885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878588.029894:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878588/real 1788878588] req@ffffa061888adc00 x1875771511517696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878588.029915:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511517696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878588.029921:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878588.029923:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878588.029926:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878588.029948:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878588.029961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878588.029969:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878588/real 1788878588] req@ffffa061888afb80 x1875771511517824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878588.030005:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511517824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878588.030012:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878588.030015:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878588.030017:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878588.030023:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878588.030040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878588.030049:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878588/real 1788878588] req@ffffa061888aea00 x1875771511517952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878588.030058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511517952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878588.030065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878588.030067:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878588.030070:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511517952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878588.030074:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878588.030085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878588.030092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878588/real 1788878588] req@ffffa061888af100 x1875771511518080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878588.030101:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511518080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878588.030107:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878588.030109:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878588.030112:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878588.669502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878588.669605:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878588.925500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878589.693510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878589.693634:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878589.949508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878590.717466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878590.717553:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878590.973484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878591.741503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878591.741560:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878591.997508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878592.765467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878592.765539:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878593.021500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878593.149501:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878593.149509:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878593.149513:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878593.149523:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3579 00000100:00080000:2.0:1788878593.149535:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878593.149553:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878593.149589:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511518208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878593.149622:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878593.149624:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878593.149626:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878593.149629:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3579 00000100:00080000:2.0:1788878593.149631:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878593.149633:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878593.149648:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878593.149660:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511518336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878593.149667:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511518208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878593.149672:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878593.149686:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878593.149688:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878593.149690:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3579 00000100:00080000:2.0:1788878593.149693:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878593.149695:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878593.149699:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878593.149707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511518464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878593.149714:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878593.149717:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878593.149719:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878593.149721:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3579 00000100:00080000:2.0:1788878593.149735:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878593.149738:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878593.149751:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878593.149759:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878593.149761:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511518592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878593.149769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878593.149772:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878593.149774:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878593.149776:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3579 00000100:00080000:2.0:1788878593.149778:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878593.149781:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878593.149784:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878593.149792:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511518720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878593.149800:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878593/real 1788878593] req@ffffa061888ae680 x1875771511518208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878593.149834:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511518208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878593.149847:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878593.149851:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878593.149854:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511518208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878593.149874:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878593.149904:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878593.149913:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878593/real 1788878593] req@ffffa061888ac380 x1875771511518336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878593.149921:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511518336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878593.149927:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878593.149930:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878593.149932:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878593.149935:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878593.149971:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878593.149991:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878593/real 1788878593] req@ffffa061888ad880 x1875771511518464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878593.150005:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511518464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878593.150020:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878593.150023:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878593.150025:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878593.150030:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878593.150057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878593.150064:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878593/real 1788878593] req@ffffa061888aed80 x1875771511518592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878593.150072:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511518592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878593.150078:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878593.150081:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878593.150085:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878593.150091:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878593.150109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878593.150121:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878593/real 1788878593] req@ffffa061888ac700 x1875771511518720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878593.150133:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511518720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878593.150138:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878593.150140:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878593.150143:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878593.789494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878593.789703:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878594.045596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878594.813484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878594.813543:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878595.069579:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878595.837507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878595.837719:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878596.093534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878596.861532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878596.861608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878597.117559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878597.885561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878597.885617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878598.141481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878598.269522:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878598.269537:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878598.269545:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878598.269574:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3584 00000100:00080000:2.0:1788878598.269590:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878598.269601:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878598.269654:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511518848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878598.269685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878598.269691:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878598.269694:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878598.269700:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3584 00000100:00080000:2.0:1788878598.269705:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878598.269710:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878598.269722:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511518848:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878598.269724:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878598.269769:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511518976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878598.269786:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878598.269791:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878598.269794:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878598.269799:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3584 00000100:00080000:2.0:1788878598.269816:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878598.269822:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878598.269828:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788878598.269844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878598.269851:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511519104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878598.269880:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878598.269888:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878598.269892:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878598.269902:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3584 00000100:00080000:2.0:1788878598.269931:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878598.269955:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878598.269955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878598/real 1788878598] req@ffffa061888ad180 x1875771511518848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878598.269978:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878598.269995:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511518848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878598.270020:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511519232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878598.270025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878598.270041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878598.270052:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511518848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878598.270055:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878598.270067:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878598.270072:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878598.270082:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3584 00000100:00100000:3.0:1788878598.270091:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518976:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878598.270094:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878598.270122:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878598.270134:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878598.270147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878598.270166:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511519360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878598.270183:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878598/real 1788878598] req@ffffa061888ace00 x1875771511518976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878598.270220:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511518976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878598.270247:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878598.270252:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878598.270257:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511518976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878598.270266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878598.270298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878598.270315:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878598/real 1788878598] req@ffffa061888ac700 x1875771511519104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878598.270339:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511519104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878598.270349:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878598.270353:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878598.270357:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878598.270364:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878598.270433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878598.270449:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878598/real 1788878598] req@ffffa061888aed80 x1875771511519232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878598.270464:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511519232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878598.270474:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878598.270478:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878598.270482:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878598.270506:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878598.270539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878598.270551:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878598/real 1788878598] req@ffffa061888ad880 x1875771511519360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878598.270564:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511519360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878598.270575:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878598.270579:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878598.270583:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878598.909458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878598.909662:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878599.165507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878599.933468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878599.933576:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878600.189507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878600.957543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878600.957713:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878601.213460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878601.981480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878601.981641:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878602.237477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878603.005542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878603.005666:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878603.261457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878603.389537:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878603.389548:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878603.389564:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878603.389575:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3590 00000100:00080000:3.0:1788878603.389588:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878603.389597:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878603.389639:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511519488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878603.389669:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878603.389673:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878603.389675:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878603.389678:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3590 00000100:00080000:3.0:1788878603.389682:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878603.389686:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878603.389698:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878603.389723:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511519488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878603.389734:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511519616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878603.389746:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878603.389749:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878603.389752:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878603.389756:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3590 00000100:00080000:3.0:1788878603.389760:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878603.389763:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878603.389768:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878603.389779:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511519744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878603.389797:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878603.389800:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878603.389802:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878603.389804:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3590 00000100:00080000:3.0:1788878603.389807:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878603.389810:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878603.389814:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878603.389819:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878603.389829:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511519872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878603.389838:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878603.389842:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878603.389843:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878603.389847:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3590 00000100:00080000:3.0:1788878603.389851:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878603.389855:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878603.389859:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878603.389865:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878603/real 1788878603] req@ffffa06188872680 x1875771511519488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878603.389885:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511520000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878603.389931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511519488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878603.389943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878603.389947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878603.389951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511519488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878603.389974:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878603.389995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878603.390005:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878603/real 1788878603] req@ffffa06188873800 x1875771511519616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878603.390034:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511519616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878603.390044:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878603.390048:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878603.390052:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878603.390058:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878603.390086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878603.390096:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878603/real 1788878603] req@ffffa06188872300 x1875771511519744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878603.390108:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511519744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878603.390116:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878603.390120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878603.390123:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878603.390128:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878603.390173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878603.390183:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878603/real 1788878603] req@ffffa06188873b80 x1875771511519872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878603.390193:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511519872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878603.390199:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878603.390202:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878603.390205:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511519872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878603.390210:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878603.390224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878603.390232:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878603/real 1788878603] req@ffffa06188870700 x1875771511520000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878603.390241:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511520000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878603.390248:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878603.390250:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878603.390252:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878604.029501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878604.029620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878604.285445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878605.053549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878605.053655:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878605.309545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878606.077558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878606.077614:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878606.333515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878607.101577:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878607.101676:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878607.357534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878608.125543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878608.125690:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878608.381525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878608.509584:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878608.509601:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878608.509610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878608.509630:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3595 00000100:00080000:1.0:1788878608.509651:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878608.509663:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878608.509736:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511520128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878608.509791:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878608.509799:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878608.509803:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878608.509812:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3595 00000100:00080000:1.0:1788878608.509821:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878608.509830:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878608.509881:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878608.509912:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511520256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878608.509919:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511520128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878608.509936:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878608.509946:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878608.509950:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878608.509956:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3595 00000100:00080000:1.0:1788878608.509963:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878608.509971:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878608.509978:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878608.509995:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511520384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878608.510009:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878608.510014:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878608.510017:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878608.510022:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3595 00000100:00080000:1.0:1788878608.510038:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878608.510043:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878608.510055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788878608.510056:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878608.510077:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511520512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878608.510093:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878608.510100:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878608.510104:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878608.510114:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3595 00000100:00080000:1.0:1788878608.510124:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878608.510132:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878608.510134:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878608/real 1788878608] req@ffffa0618869b100 x1875771511520128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878608.510143:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878608.510170:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511520640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878608.510189:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511520128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878608.510217:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878608.510226:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878608.510233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511520128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878608.510270:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878608.510302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878608.510318:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878608/real 1788878608] req@ffffa06188698380 x1875771511520256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878608.510333:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511520256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878608.510344:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878608.510348:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878608.510353:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878608.510360:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878608.510380:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878608.510432:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878608/real 1788878608] req@ffffa06188699c00 x1875771511520384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878608.510448:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511520384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878608.510459:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878608.510465:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878608.510471:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878608.510486:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878608.510529:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878608.510550:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878608/real 1788878608] req@ffffa0618869b800 x1875771511520512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878608.510572:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511520512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878608.510589:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878608.510595:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878608.510602:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878608.510615:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878608.510652:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878608.510674:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878608/real 1788878608] req@ffffa0618869aa00 x1875771511520640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878608.510694:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511520640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878608.510710:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878608.510716:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878608.510722:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878609.149558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878609.149721:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878609.405567:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878610.173497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878610.173624:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878610.429498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878611.197553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878611.197730:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878611.453555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878612.221449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878612.221511:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878612.477529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878613.245493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878613.245651:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878613.501491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878613.629567:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878613.629596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878613.629603:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878613.629622:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3600 00000100:00080000:1.0:1788878613.629640:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878613.629656:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878613.629728:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511520768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878613.629763:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878613.629766:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878613.629768:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878613.629771:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3600 00000100:00080000:1.0:1788878613.629774:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878613.629777:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878613.629787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878613.629797:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511520896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878613.629834:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878613.629837:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878613.629839:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878613.629843:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3600 00000100:00080000:1.0:1788878613.629847:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878613.629850:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878613.629854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878613.629864:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511521024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878613.629872:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878613.629875:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878613.629877:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878613.629883:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3600 00000100:00100000:0.0:1788878613.629886:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511520768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878613.629887:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878613.629892:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878613.629904:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878613.629914:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511521152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878613.629921:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878613.629923:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878613.629925:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878613.629928:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3600 00000100:00080000:1.0:1788878613.629931:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878613.629933:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878613.629936:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878613.629944:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511521280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878613.630019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878613.630082:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878613/real 1788878613] req@ffffa0618869a680 x1875771511520768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878613.630104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511520768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878613.630127:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878613.630131:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878613.630136:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511520768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878613.630146:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878613.630168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878613.630191:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878613/real 1788878613] req@ffffa0618869bb80 x1875771511520896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878613.630202:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511520896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878613.630211:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878613.630214:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878613.630218:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511520896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878613.630258:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878613.630292:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878613.630310:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878613/real 1788878613] req@ffffa0618869ad80 x1875771511521024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878613.630330:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511521024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878613.630343:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878613.630348:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878613.630353:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878613.630363:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878613.630427:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878613.630457:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878613/real 1788878613] req@ffffa06188698000 x1875771511521152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878613.630476:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511521152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878613.630490:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878613.630495:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878613.630510:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878613.630521:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878613.630554:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878613.630581:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878613/real 1788878613] req@ffffa06188699f80 x1875771511521280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878613.630601:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511521280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878613.630609:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878613.630613:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878613.630616:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878614.269554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878614.269663:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878614.525555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878615.293457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878615.293540:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878615.549498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878616.317542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878616.317645:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878616.573504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878617.341508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878617.341627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878617.597527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878618.365592:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878618.365628:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878618.621559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878618.749526:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878618.749540:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878618.749546:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878618.749562:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3605 00000100:00080000:2.0:1788878618.749580:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878618.749592:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878618.749644:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511521408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878618.749682:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878618.749688:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878618.749691:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878618.749696:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3605 00000100:00080000:2.0:1788878618.749701:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878618.749707:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878618.749720:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878618.749745:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511521408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878618.749746:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511521536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878618.749811:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878618.749819:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878618.749839:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878618.749848:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3605 00000100:00080000:2.0:1788878618.749858:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878618.749867:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878618.749877:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788878618.749881:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878618.749910:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511521664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878618.749925:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878618.749930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878618/real 1788878618] req@ffffa061888ac380 x1875771511521408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878618.749932:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878618.749939:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878618.749953:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3605 00000100:00080000:2.0:1788878618.749966:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878618.749974:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878618.749981:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511521408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878618.749988:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878618.750010:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878618.750019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788878618.750028:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511521792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878618.750031:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511521408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878618.750063:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878618.750071:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878618.750080:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878618.750081:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521536:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878618.750096:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3605 00000100:00080000:2.0:1788878618.750104:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878618.750111:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878618.750121:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878618.750129:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878618.750151:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511521920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878618.750155:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878618/real 1788878618] req@ffffa061888ae680 x1875771511521536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878618.750209:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511521536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878618.750222:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878618.750227:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878618.750232:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878618.750241:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878618.750263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878618.750276:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878618/real 1788878618] req@ffffa061888af100 x1875771511521664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878618.750290:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511521664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878618.750300:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878618.750304:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878618.750309:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878618.750320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878618.750356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878618.750378:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878618/real 1788878618] req@ffffa061888aea00 x1875771511521792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878618.750444:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511521792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878618.750456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878618.750460:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878618.750464:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878618.750487:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878618.750516:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878618.750531:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878618/real 1788878618] req@ffffa061888afb80 x1875771511521920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878618.750545:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511521920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878618.750555:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878618.750559:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878618.750563:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511521920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878619.389501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878619.389641:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878619.645487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878620.413560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878620.413620:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878620.669542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878621.437632:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878621.437668:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878621.693465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878622.461473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878622.461590:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878622.717547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878623.485501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878623.485649:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878623.741482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878623.869534:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878623.869550:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878623.869559:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878623.869578:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3610 00000100:00080000:2.0:1788878623.869596:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878623.869612:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878623.869682:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511522048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878623.869706:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878623.869710:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878623.869711:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878623.869714:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3610 00000100:00080000:2.0:1788878623.869718:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878623.869722:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878623.869744:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878623.869747:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511522048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878623.869758:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511522176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878623.869767:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878623.869769:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878623.869771:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878623.869775:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3610 00000100:00080000:2.0:1788878623.869778:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878623.869781:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878623.869785:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878623.869794:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511522304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878623.869800:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878623.869803:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878623.869804:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878623.869807:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3610 00000100:00080000:2.0:1788878623.869809:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878623.869812:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878623.869815:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878623.869819:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878623.869826:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511522432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878623.869833:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878623.869853:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878623.869855:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878623.869860:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3610 00000100:00080000:2.0:1788878623.869865:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878623.869870:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878623.869876:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878623.869882:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878623/real 1788878623] req@ffffa061888adc00 x1875771511522048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878623.869892:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511522560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878623.869919:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511522048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878623.869932:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878623.869936:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878623.869941:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511522048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878623.869966:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878623.870041:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878623.870051:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878623/real 1788878623] req@ffffa061888ae300 x1875771511522176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878623.870059:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511522176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878623.870065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878623.870068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878623.870070:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878623.870074:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878623.870086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878623.870092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878623/real 1788878623] req@ffffa061888af480 x1875771511522304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878623.870100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511522304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878623.870105:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878623.870107:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878623.870109:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878623.870112:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878623.870123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878623.870129:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878623/real 1788878623] req@ffffa061888aca80 x1875771511522432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878623.870136:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511522432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878623.870141:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878623.870143:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878623.870145:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878623.870148:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878623.870157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878623.870163:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878623/real 1788878623] req@ffffa061888ac000 x1875771511522560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878623.870169:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511522560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878623.870179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878623.870181:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878623.870183:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878624.509519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878624.509642:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878624.765570:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878625.533516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878625.533673:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878625.789520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878626.557592:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878626.557784:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878626.813522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878627.581528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878627.581674:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878627.837493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878628.605575:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878628.605635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878628.861504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878628.989615:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878628.989639:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878628.989645:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878628.989661:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3615 00000100:00080000:3.0:1788878628.989677:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878628.989689:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878628.989774:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511522688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878628.989810:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878628.989815:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878628.989818:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878628.989823:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3615 00000100:00080000:3.0:1788878628.989828:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878628.989834:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878628.989847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878628.989865:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511522816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878628.989875:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511522688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878628.989892:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878628.989898:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878628.989900:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878628.989906:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3615 00000100:00080000:3.0:1788878628.989910:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878628.989916:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878628.989922:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878628.989938:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511522944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878628.989951:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878628.989955:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878628.989958:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878628.989962:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3615 00000100:00080000:3.0:1788878628.989967:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878628.989971:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878628.989977:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878628.989992:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511523072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878628.990008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878628.990024:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878628.990032:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878628.990035:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878628.990042:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3615 00000100:00080000:3.0:1788878628.990047:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878628.990053:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878628.990060:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878628.990077:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511523200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878628.990084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878628/real 1788878628] req@ffffa06188872d80 x1875771511522688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878628.990152:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511522688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878628.990168:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878628.990173:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878628.990179:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511522688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878628.990213:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878628.990248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878628.990285:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878628/real 1788878628] req@ffffa06188870a80 x1875771511522816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878628.990312:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511522816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878628.990330:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878628.990337:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878628.990344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878628.990356:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878628.990420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878628.990439:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878628/real 1788878628] req@ffffa06188870380 x1875771511522944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878628.990455:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511522944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878628.990465:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878628.990469:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878628.990473:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511522944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878628.990481:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878628.990504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878628.990515:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878628/real 1788878628] req@ffffa06188871c00 x1875771511523072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878628.990529:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511523072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878628.990540:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878628.990544:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878628.990548:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878628.990554:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878628.990573:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878628.990583:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878628/real 1788878628] req@ffffa06188871880 x1875771511523200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878628.990597:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511523200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878628.990607:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878628.990610:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878628.990614:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878629.629510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878629.629645:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878629.885571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878630.653461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878630.653568:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878630.909453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878631.677554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878631.677635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878631.933496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878632.701474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878632.701595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878632.957547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878633.725544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878633.725649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878633.981537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878634.109564:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878634.109574:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878634.109580:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878634.109593:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3620 00000100:00080000:2.0:1788878634.109606:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878634.109615:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878634.109678:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511523328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878634.109707:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878634.109711:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878634.109713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878634.109718:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3620 00000100:00080000:2.0:1788878634.109722:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878634.109727:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878634.109740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878634.109756:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511523456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878634.109762:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511523328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878634.109780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878634.109784:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878634.109786:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878634.109792:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3620 00000100:00080000:2.0:1788878634.109797:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878634.109800:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878634.109806:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878634.109821:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511523584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878634.109830:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878634.109833:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878634.109835:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878634.109839:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3620 00000100:00080000:2.0:1788878634.109842:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878634.109845:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878634.109851:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878634.109856:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878634.109867:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511523712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878634.109878:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878634.109881:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878634.109883:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878634.109887:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3620 00000100:00080000:2.0:1788878634.109891:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878634.109895:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878634.109900:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878634.109906:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878634/real 1788878634] req@ffffa061888adf80 x1875771511523328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878634.109916:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511523840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878634.109934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511523328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878634.109948:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878634.109953:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878634.109958:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511523328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878634.109982:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878634.110004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878634.110015:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878634/real 1788878634] req@ffffa061888ad500 x1875771511523456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878634.110026:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511523456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878634.110034:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878634.110037:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878634.110040:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878634.110045:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878634.110059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878634.110067:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878634/real 1788878634] req@ffffa061888ac000 x1875771511523584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878634.110083:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511523584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878634.110090:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878634.110093:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878634.110095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878634.110100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878634.110114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878634.110122:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878634/real 1788878634] req@ffffa061888aca80 x1875771511523712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878634.110132:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511523712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878634.110139:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878634.110141:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878634.110144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878634.110148:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878634.110161:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878634.110169:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878634/real 1788878634] req@ffffa061888af480 x1875771511523840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878634.110178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511523840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878634.110191:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878634.110194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878634.110197:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511523840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878634.749489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878634.749702:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878635.005521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878635.773576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878635.773718:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878636.029515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878636.797500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878636.797576:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878637.053552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878637.821528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878637.821712:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878638.077523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878638.845523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878638.845686:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878639.101561:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878639.229507:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878639.229518:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878639.229524:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878639.229537:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3625 00000100:00080000:2.0:1788878639.229552:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878639.229562:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878639.229608:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511523968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878639.229671:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878639.229675:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878639.229677:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878639.229682:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3625 00000100:00080000:2.0:1788878639.229686:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878639.229691:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878639.229704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878639.229722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511524096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878639.229748:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511523968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878639.229768:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878639.229771:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878639.229773:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878639.229778:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3625 00000100:00080000:2.0:1788878639.229783:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878639.229787:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878639.229793:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878639.229807:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511524224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878639.229817:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878639.229820:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878639.229823:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878639.229827:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3625 00000100:00080000:2.0:1788878639.229830:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878639.229834:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878639.229838:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878639.229849:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511524352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878639.229860:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878639.229864:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878639.229876:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878639.229881:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3625 00000100:00080000:2.0:1788878639.229884:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878639.229887:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878639.229892:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878639.229899:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878639.229905:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511524480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878639.229976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878639.230018:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878639.230078:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878639.869536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878639.869609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878640.125504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878640.893520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878640.893607:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878641.149501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878641.917534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878641.917654:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878642.173482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878642.301784:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878642.301822:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878642.301836:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878642.301881:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878642.301889:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878642.301898:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878642.301905:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878642.301915:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878639/real 1788878642] req@ffffa061888adc00 x1875771511524096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878642.301944:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511524096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878642.301959:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.301964:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878642.301969:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878642.301978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878639/real 1788878642] req@ffffa061888afb80 x1875771511524224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878642.301991:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511524224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878642.302011:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.302015:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878642.302018:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878642.302026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878639/real 1788878642] req@ffffa061888aea00 x1875771511524352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878642.302039:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511524352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878642.302047:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.302051:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878642.302054:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878642.302064:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878639/real 1788878642] req@ffffa061888ae300 x1875771511523968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878642.302076:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511523968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878642.302083:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.302086:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878642.302090:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511523968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878642.302118:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878639/real 1788878642] req@ffffa061888af100 x1875771511524480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878642.302139:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511524480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878642.302148:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.302152:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878642.302167:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878642.941463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878642.941543:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878643.197481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878643.965548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878643.965697:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878644.221473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878644.349573:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878644.349610:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878644.349618:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878644.349647:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3631 00000100:00080000:2.0:1788878644.349668:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878644.349684:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878644.349749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511524608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878644.349784:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878644.349792:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878644.349797:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878644.349805:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3631 00000100:00080000:2.0:1788878644.349814:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878644.349824:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878644.349843:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878644.349848:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511524608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878644.349877:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511524736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878644.349902:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878644.349907:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878644.349910:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878644.349917:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3631 00000100:00080000:2.0:1788878644.349927:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878644.349936:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878644.349947:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878644.349981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878644.349983:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511524864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878644.350006:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878644.350012:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878644.350016:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878644.350025:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3631 00000100:00080000:2.0:1788878644.350035:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878644.350044:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878644.350054:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878644.350087:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878644/real 1788878644] req@ffffa061888ae680 x1875771511524608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878644.350088:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511524992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878644.350162:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878644.350172:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878644.350175:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878644.350184:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3631 00000100:00100000:0.0:1788878644.350192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511524608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878644.350196:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878644.350215:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878644.350233:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878644.350236:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878644.350250:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878644.350259:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511524608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878644.350277:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511525120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878644.350308:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878644.350378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878644.350459:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878644/real 1788878644] req@ffffa061888ac380 x1875771511524736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878644.350485:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511524736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878644.350500:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878644.350506:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878644.350513:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878644.350524:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878644.350564:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878644.350586:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878644/real 1788878644] req@ffffa061888ad880 x1875771511524864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878644.350641:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511524864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878644.350678:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878644.350684:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878644.350691:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878644.350704:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878644.350746:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878644.350769:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878644/real 1788878644] req@ffffa061888aed80 x1875771511524992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878644.350794:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511524992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878644.350811:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878644.350816:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878644.350823:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511524992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878644.350858:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878644.350898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878644.350920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878644/real 1788878644] req@ffffa061888ac700 x1875771511525120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878644.350942:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511525120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878644.350956:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878644.350962:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878644.350970:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878644.989500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878644.989716:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878645.245474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878646.013517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878646.013639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878646.269621:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878647.037498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878647.037595:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878647.293500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878648.061537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878648.061639:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878648.317626:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878649.085456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878649.085521:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878649.341497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878649.469510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878649.469518:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878649.469522:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878649.469531:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3636 00000100:00080000:3.0:1788878649.469541:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878649.469549:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878649.469598:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511525248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878649.469621:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878649.469624:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878649.469625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878649.469628:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3636 00000100:00080000:3.0:1788878649.469631:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878649.469633:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878649.469642:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878649.469651:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511525376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878649.469659:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878649.469663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878649.469665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878649.469668:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3636 00000100:00080000:3.0:1788878649.469670:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878649.469672:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878649.469675:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511525248:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878649.469675:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878649.469683:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511525504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878649.469691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878649.469693:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878649.469694:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878649.469697:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3636 00000100:00080000:3.0:1788878649.469698:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878649.469700:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878649.469704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878649.469711:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511525632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878649.469717:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878649.469719:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878649.469720:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878649.469722:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3636 00000100:00080000:3.0:1788878649.469725:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878649.469728:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878649.469731:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878649.469738:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511525760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878649.469753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878649.469789:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878649/real 1788878649] req@ffffa06188871880 x1875771511525248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878649.469809:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511525248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878649.469821:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878649.469825:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878649.469829:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511525248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878649.469854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878649.469879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878649.469892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878649/real 1788878649] req@ffffa06188871c00 x1875771511525376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878649.469905:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511525376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878649.469914:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878649.469918:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878649.469922:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878649.469929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878649.469950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878649.469963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878649/real 1788878649] req@ffffa06188870380 x1875771511525504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878649.469974:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511525504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878649.469982:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878649.469985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878649.469989:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878649.469995:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878649.470015:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878649.470027:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878649/real 1788878649] req@ffffa06188870a80 x1875771511525632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878649.470039:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511525632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878649.470047:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878649.470051:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878649.470055:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878649.470061:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878649.470080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878649.470092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878649/real 1788878649] req@ffffa06188872d80 x1875771511525760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878649.470104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511525760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878649.470113:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878649.470116:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878649.470120:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511525760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878650.109466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878650.109530:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878650.365525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878651.133533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878651.133639:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878651.389492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878652.157529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878652.157680:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878652.413476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878653.181499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878653.181708:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878653.437544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878654.205466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878654.205530:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878654.461457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878654.589463:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878654.589470:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878654.589473:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878654.589482:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3641 00000100:00080000:0.0:1788878654.589491:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878654.589500:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878654.589555:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511525888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878654.589584:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878654.589588:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878654.589590:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878654.589595:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3641 00000100:00080000:0.0:1788878654.589600:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878654.589605:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878654.589617:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511525888:192.168.203.111@tcp:250:kworker.0 00000008:00100000:0.0:1788878654.589618:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878654.589633:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511526016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878654.589642:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878654.589645:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878654.589646:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878654.589649:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3641 00000100:00080000:0.0:1788878654.589652:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878654.589655:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878654.589659:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878654.589666:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511526144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878654.589673:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878654.589675:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878654.589677:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878654.589682:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3641 00000800:00000100:3.0:1788878654.589683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878654.589687:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878654.589691:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878654.589695:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878654.589703:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511526272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878654.589709:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878654.589711:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878654.589712:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878654.589716:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3641 00000100:00080000:0.0:1788878654.589718:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878654.589721:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878654/real 1788878654] req@ffffa06198afed80 x1875771511525888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878654.589724:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878654.589730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878654.589740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511526400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878654.589751:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511525888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878654.589760:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878654.589764:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878654.589768:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511525888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878654.589794:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878654.589814:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878654.589822:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878654/real 1788878654] req@ffffa06198aff480 x1875771511526016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878654.589830:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511526016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878654.589836:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878654.589838:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878654.589840:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878654.589843:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878654.589854:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878654.589859:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878654/real 1788878654] req@ffffa06198afd880 x1875771511526144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878654.589866:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511526144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878654.589876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878654.589878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878654.589880:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878654.589883:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878654.589893:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878654.589899:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878654/real 1788878654] req@ffffa06198affb80 x1875771511526272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878654.589906:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511526272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878654.589911:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878654.589913:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878654.589915:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878654.589918:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878654.589927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878654.589933:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878654/real 1788878654] req@ffffa06198aff800 x1875771511526400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878654.589941:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511526400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878654.589945:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878654.589947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878654.589950:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878655.229469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878655.229561:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878655.485525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878656.253453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878656.253574:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878656.509487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878657.277473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878657.277551:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878657.533462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878658.301481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878658.301537:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878658.557504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878659.325477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878659.325556:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878659.581476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878659.709520:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878659.709530:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878659.709535:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878659.709547:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3646 00000100:00080000:1.0:1788878659.709558:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878659.709567:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878659.709608:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511526528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878659.709647:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878659.709651:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878659.709653:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878659.709658:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3646 00000100:00080000:1.0:1788878659.709662:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878659.709669:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878659.709680:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878659.709697:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511526656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878659.709710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878659.709718:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788878659.709718:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511526528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878659.709721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878659.709736:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3646 00000100:00080000:1.0:1788878659.709741:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878659.709747:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878659.709754:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878659.709770:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511526784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878659.709781:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878659.709786:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878659.709789:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878659.709793:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3646 00000100:00080000:1.0:1788878659.709798:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788878659.709801:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878659.709808:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878659.709817:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878659.709833:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511526912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878659.709845:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878659/real 1788878659] req@ffffa06188699f80 x1875771511526528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878659.709847:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878659.709864:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878659.709871:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878659.709880:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3646 00000100:00100000:3.0:1788878659.709882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511526528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878659.709891:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878659.709905:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878659.709908:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:3.0:1788878659.709913:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878659.709919:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511526528:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878659.709920:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878659.709943:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511527040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878659.709950:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878659.709978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878659.709989:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878659/real 1788878659] req@ffffa06188698000 x1875771511526656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878659.710000:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511526656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878659.710008:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878659.710010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878659.710013:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878659.710017:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878659.710028:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878659.710035:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878659/real 1788878659] req@ffffa0618869ad80 x1875771511526784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878659.710043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511526784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878659.710048:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878659.710050:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878659.710052:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878659.710055:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878659.710067:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878659.710073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878659/real 1788878659] req@ffffa0618869bb80 x1875771511526912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878659.710080:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511526912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878659.710085:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878659.710087:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878659.710089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511526912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878659.710100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878659.710110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878659.710116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878659/real 1788878659] req@ffffa0618869a680 x1875771511527040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878659.710123:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511527040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878659.710130:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878659.710132:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878659.710134:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878660.349458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878660.349538:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878660.605463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878661.373489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878661.373530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878661.629476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878662.397457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878662.397510:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878662.653455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878663.421498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878663.421589:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878663.677450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878664.445587:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878664.445653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878664.701468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878664.829475:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878664.829484:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878664.829488:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878664.829499:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3651 00000100:00080000:2.0:1788878664.829510:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878664.829517:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878664.829558:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511527168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878664.829580:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878664.829583:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878664.829585:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878664.829588:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3651 00000100:00080000:2.0:1788878664.829591:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878664.829593:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878664.829601:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878664.829611:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511527296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878664.829619:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878664.829623:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878664.829624:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878664.829627:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3651 00000100:00080000:2.0:1788878664.829630:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878664.829632:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878664.829635:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878664.829638:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511527168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878664.829646:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511527424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878664.829654:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878664.829656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878664.829658:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878664.829661:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3651 00000100:00080000:2.0:1788878664.829663:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878664.829666:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878664.829669:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878664.829677:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511527552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878664.829683:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878664.829686:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878664.829687:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878664.829689:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3651 00000100:00080000:2.0:1788878664.829693:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878664.829696:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878664.829700:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878664.829710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511527680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878664.829710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878664.829747:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878664/real 1788878664] req@ffffa061888ace00 x1875771511527168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878664.829763:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511527168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878664.829772:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878664.829775:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878664.829778:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511527168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878664.829783:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878664.829797:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878664.829805:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878664/real 1788878664] req@ffffa061888ad180 x1875771511527296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878664.829813:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511527296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878664.829819:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878664.829821:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878664.829823:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878664.829845:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878664.829857:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878664.829863:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878664/real 1788878664] req@ffffa061888ac700 x1875771511527424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878664.829871:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511527424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878664.829876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878664.829878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878664.829880:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878664.829884:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878664.829895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878664.829900:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878664/real 1788878664] req@ffffa061888aed80 x1875771511527552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878664.829908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511527552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878664.829913:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878664.829915:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878664.829917:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878664.829920:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878664.829930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878664.829935:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878664/real 1788878664] req@ffffa061888ad880 x1875771511527680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878664.829943:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511527680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878664.829949:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878664.829950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878664.829952:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878665.469570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878665.469663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878665.725523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878666.493539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878666.493689:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878666.749533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878667.517575:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878667.517757:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878667.773596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878668.541559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878668.541632:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878668.797537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878669.565539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878669.565639:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878669.821528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878669.949592:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878669.949607:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878669.949623:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878669.949640:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3656 00000100:00080000:3.0:1788878669.949658:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878669.949670:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878669.949741:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511527808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878669.949784:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878669.949789:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878669.949792:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878669.949797:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3656 00000100:00080000:3.0:1788878669.949802:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878669.949807:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878669.949822:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878669.949847:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511527936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878669.949850:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511527808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878669.949888:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878669.949892:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878669.949895:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878669.949901:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3656 00000100:00080000:3.0:1788878669.949906:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878669.949910:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878669.949916:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878669.949932:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511528064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878669.949953:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878669.949958:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878669.949961:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878669.949966:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3656 00000100:00080000:3.0:1788878669.949971:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878669.949975:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878669.949981:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878669.949997:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511528192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878669.950005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878669.950012:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878669.950018:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878669.950021:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878669.950027:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3656 00000100:00080000:3.0:1788878669.950033:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878669.950039:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878669.950045:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878669.950060:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511528320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878669.950089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878669/real 1788878669] req@ffffa06188870700 x1875771511527808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878669.950121:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511527808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878669.950137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878669.950144:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878669.950150:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511527808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878669.950185:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878669.950219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878669.950247:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878669/real 1788878669] req@ffffa06188873b80 x1875771511527936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878669.950264:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511527936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878669.950275:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878669.950280:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878669.950284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511527936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878669.950292:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878669.950315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878669.950328:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878669/real 1788878669] req@ffffa06188872300 x1875771511528064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878669.950343:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511528064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878669.950353:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878669.950357:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878669.950360:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878669.950368:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878669.950457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878669.950476:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878669/real 1788878669] req@ffffa06188873800 x1875771511528192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878669.950493:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511528192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878669.950503:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878669.950507:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878669.950511:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878669.950518:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878669.950542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878669.950553:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878669/real 1788878669] req@ffffa06188872680 x1875771511528320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878669.950568:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511528320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878669.950579:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878669.950584:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878669.950591:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878670.589560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878670.589695:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878670.845561:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878671.613582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878671.613651:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878671.869596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878672.637511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878672.637627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878672.893480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878673.661489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878673.661550:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878673.917555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878674.685482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878674.685547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878674.941574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878675.069622:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878675.069638:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878675.069645:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878675.069660:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3661 00000100:00080000:2.0:1788878675.069678:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878675.069689:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878675.069750:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511528448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878675.069797:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878675.069803:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878675.069805:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878675.069811:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3661 00000100:00080000:2.0:1788878675.069816:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878675.069822:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878675.069838:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878675.069855:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511528576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878675.069874:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878675.069887:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788878675.069888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511528448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878675.069891:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878675.069915:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3661 00000100:00080000:2.0:1788878675.069921:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878675.069927:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878675.069936:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878675.069963:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511528704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878675.069975:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878675.069980:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878675.069982:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878675.069989:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3661 00000100:00080000:2.0:1788878675.069994:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878675.069998:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878675.070005:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878675.070020:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511528832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878675.070034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878675.070050:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878675.070057:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878675.070062:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878675.070068:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3661 00000100:00080000:2.0:1788878675.070074:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878675.070080:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878675.070089:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878675.070111:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878675/real 1788878675] req@ffffa061888ac380 x1875771511528448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878675.070112:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511528960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878675.070255:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511528448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878675.070278:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878675.070285:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878675.070291:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511528448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878675.070325:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878675.070356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878675.070371:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878675/real 1788878675] req@ffffa061888ae680 x1875771511528576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878675.070445:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511528576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878675.070457:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878675.070462:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878675.070467:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878675.070476:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878675.070502:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878675.070516:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878675/real 1788878675] req@ffffa061888af100 x1875771511528704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878675.070532:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511528704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878675.070542:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878675.070546:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878675.070550:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878675.070557:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878675.070578:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878675.070590:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878675/real 1788878675] req@ffffa061888ae300 x1875771511528832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878675.070604:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511528832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878675.070614:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878675.070618:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878675.070621:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878675.070629:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878675.070648:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878675.070659:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878675/real 1788878675] req@ffffa061888aea00 x1875771511528960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878675.070673:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511528960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878675.070691:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878675.070695:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878675.070699:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511528960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878675.709568:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878675.709683:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878675.965535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878676.733509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878676.733594:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878676.989631:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878677.757505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878677.757630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878678.013489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878678.781517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878678.781597:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878679.037524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878679.805502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878679.805575:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878680.061479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878680.189504:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878680.189518:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878680.189523:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878680.189533:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3666 00000100:00080000:3.0:1788878680.189545:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878680.189552:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878680.189592:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511529088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878680.189620:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878680.189623:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878680.189625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878680.189628:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3666 00000100:00080000:3.0:1788878680.189630:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878680.189633:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878680.189643:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878680.189653:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511529216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878680.189664:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511529088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878680.189668:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878680.189682:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878680.189684:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878680.189688:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3666 00000100:00080000:3.0:1788878680.189691:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878680.189694:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878680.189698:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878680.189707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511529344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878680.189714:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878680.189718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878680.189720:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878680.189724:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3666 00000100:00080000:3.0:1788878680.189729:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878680.189731:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878680.189735:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878680.189742:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511529472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878680.189749:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878680.189750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878680.189765:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878680.189768:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878680.189773:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3666 00000100:00080000:3.0:1788878680.189777:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878680.189782:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878680.189790:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878680.189796:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878680/real 1788878680] req@ffffa06188872a00 x1875771511529088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878680.189804:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511529600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878680.189838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511529088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878680.189848:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878680.189852:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878680.189859:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511529088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878680.189880:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878680.189897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878680.189918:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878680/real 1788878680] req@ffffa06188873480 x1875771511529216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878680.189933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511529216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878680.189943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878680.189947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878680.189950:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878680.189957:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878680.189980:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878680.189989:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878680/real 1788878680] req@ffffa06188871500 x1875771511529344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878680.189997:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511529344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878680.190003:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878680.190005:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878680.190007:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878680.190015:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878680.190027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878680.190033:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878680/real 1788878680] req@ffffa06188873100 x1875771511529472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878680.190040:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511529472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878680.190046:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878680.190048:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878680.190050:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878680.190053:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878680.190063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878680.190068:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878680/real 1788878680] req@ffffa06188872680 x1875771511529600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878680.190075:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511529600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878680.190081:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878680.190082:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878680.190084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878680.829500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878680.829571:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878681.085525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878681.853477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878681.853539:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878682.109466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878682.877559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878682.877665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878683.133505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878683.901501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878683.901560:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878684.157520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878684.925522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878684.925560:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878685.181445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878685.309454:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878685.309463:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878685.309467:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878685.309478:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3672 00000100:00080000:3.0:1788878685.309489:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878685.309500:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878685.309542:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511529728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878685.309562:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878685.309565:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878685.309567:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878685.309571:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3672 00000100:00080000:3.0:1788878685.309575:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878685.309579:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878685.309590:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878685.309598:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511529728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878685.309606:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511529856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878685.309631:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878685.309634:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878685.309635:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878685.309648:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3672 00000100:00080000:3.0:1788878685.309651:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878685.309655:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878685.309659:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878685.309669:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511529984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878685.309677:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878685.309680:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878685.309681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878685.309685:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3672 00000100:00080000:3.0:1788878685.309689:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878685.309691:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878685.309704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878685.309712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878685.309718:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511530112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878685.309726:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878685.309728:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878685.309730:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878685.309734:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3672 00000100:00080000:3.0:1788878685.309740:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878685.309744:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878685.309760:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878685.309770:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511530240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878685.309793:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878685/real 1788878685] req@ffffa06188873800 x1875771511529728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878685.309812:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511529728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878685.309823:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878685.309827:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878685.309830:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511529728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878685.309855:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878685.309875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878685.309884:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878685/real 1788878685] req@ffffa06188872300 x1875771511529856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878685.309893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511529856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878685.309899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878685.309902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878685.309904:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878685.309908:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878685.309920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878685.309927:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878685/real 1788878685] req@ffffa06188873b80 x1875771511529984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878685.309941:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511529984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878685.309947:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878685.309949:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878685.309951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511529984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878685.309956:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878685.309968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878685.309975:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878685/real 1788878685] req@ffffa06188870700 x1875771511530112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878685.309984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511530112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878685.309990:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878685.309992:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878685.309994:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878685.309998:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878685.310009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878685.310016:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878685/real 1788878685] req@ffffa06188872d80 x1875771511530240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878685.310024:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511530240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878685.310036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878685.310038:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878685.310041:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878685.949512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878685.949704:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878686.205512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878686.973530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878686.973616:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878687.229588:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878687.997504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878687.997561:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878688.253526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878689.021581:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878689.021627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878689.277526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878690.045576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878690.045619:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878690.301540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878690.429553:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878690.429566:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878690.429583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878690.429596:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3677 00000100:00080000:1.0:1788878690.429609:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878690.429618:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878690.429669:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511530368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878690.429710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878690.429715:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878690.429718:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878690.429726:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3677 00000100:00080000:1.0:1788878690.429732:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878690.429738:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878690.429750:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511530368:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878690.429753:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878690.429778:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511530496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878690.429796:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878690.429801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878690.429804:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878690.429809:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3677 00000100:00080000:1.0:1788878690.429826:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878690.429832:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878690.429837:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878690.429845:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878690.429865:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511530624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878690.429887:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878690/real 1788878690] req@ffffa0618869aa00 x1875771511530368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878690.429891:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878690.429903:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878690.429909:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878690.429917:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3677 00000100:00100000:0.0:1788878690.429920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511530368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878690.429925:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878690.429945:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878690.429947:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878690.429953:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878690.429959:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511530368:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878690.429961:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878690.429984:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511530752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878690.429987:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530496:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788878690.430010:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878690.430028:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878690.430028:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878690.430036:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878690.430043:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3677 00000100:00100000:0.0:1788878690.430048:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878690/real 1788878690] req@ffffa0618869b800 x1875771511530496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878690.430050:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878690.430061:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878690.430068:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878690.430069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511530496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878690.430081:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878690.430089:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788878690.430091:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511530880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878690.430093:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878690.430114:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878690.430136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878690.430146:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878690/real 1788878690] req@ffffa06188699c00 x1875771511530624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878690.430156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511530624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878690.430163:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878690.430166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878690.430169:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878690.430180:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878690.430194:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878690.430202:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878690/real 1788878690] req@ffffa06188698380 x1875771511530752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878690.430211:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511530752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878690.430219:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878690.430222:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878690.430225:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878690.430229:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878690.430241:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878690.430248:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878690/real 1788878690] req@ffffa0618869b100 x1875771511530880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878690.430257:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511530880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878690.430265:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878690.430267:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878690.430293:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511530880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878691.069494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878691.069573:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878691.325461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878692.093470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878692.093548:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878692.349491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878693.117567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878693.117651:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878693.373575:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878694.141579:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878694.141673:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878694.397518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878695.165538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878695.165636:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878695.421553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878695.549551:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878695.549561:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878695.549566:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878695.549576:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3682 00000100:00080000:1.0:1788878695.549594:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878695.549606:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878695.549656:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511531008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878695.549686:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878695.549690:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878695.549693:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878695.549698:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3682 00000100:00080000:1.0:1788878695.549703:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878695.549709:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878695.549720:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878695.549738:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511531136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878695.549739:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511531008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878695.549773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878695.549776:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878695.549777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878695.549780:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3682 00000100:00080000:1.0:1788878695.549783:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878695.549786:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878695.549790:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878695.549799:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511531264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878695.549806:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878695.549810:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878695.549811:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878695.549815:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3682 00000100:00080000:1.0:1788878695.549818:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878695.549820:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878695.549824:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878695.549835:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511531392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878695.549844:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878695.549848:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878695.549850:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878695.549852:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3682 00000100:00080000:1.0:1788878695.549867:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878695.549871:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878695.549874:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878695.549889:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511531520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878695.549891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878695.549960:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878695/real 1788878695] req@ffffa0618869b100 x1875771511531008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878695.549987:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511531008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878695.550003:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878695.550008:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878695.550014:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511531008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878695.550052:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878695.550094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878695.550110:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878695/real 1788878695] req@ffffa06188698380 x1875771511531136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878695.550136:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511531136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878695.550147:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878695.550151:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878695.550155:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878695.550163:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878695.550182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878695.550194:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878695/real 1788878695] req@ffffa06188699c00 x1875771511531264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878695.550208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511531264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878695.550226:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878695.550230:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878695.550262:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878695.550285:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878695.550315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878695.550330:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878695/real 1788878695] req@ffffa0618869b800 x1875771511531392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878695.550345:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511531392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878695.550355:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878695.550359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878695.550363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878695.550370:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878695.550431:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878695.550445:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878695/real 1788878695] req@ffffa0618869aa00 x1875771511531520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878695.550460:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511531520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878695.550470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878695.550474:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878695.550479:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878696.189453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878696.189514:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878696.445470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878697.213514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878697.213590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878697.469510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878698.237598:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878698.237764:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878698.493514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878699.261489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878699.261552:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878699.517484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878700.285517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878700.285659:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878700.541525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878700.669512:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878700.669524:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878700.669530:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878700.669543:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3687 00000100:00080000:2.0:1788878700.669556:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878700.669566:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878700.669611:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511531648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878700.669643:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878700.669647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878700.669649:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878700.669652:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3687 00000100:00080000:2.0:1788878700.669657:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878700.669662:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878700.669672:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878700.669680:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511531648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878700.669687:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511531776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878700.669698:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878700.669700:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878700.669702:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878700.669705:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3687 00000100:00080000:2.0:1788878700.669708:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878700.669711:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878700.669715:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878700.669726:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511531904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878700.669734:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878700.669736:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878700.669738:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878700.669741:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3687 00000100:00080000:2.0:1788878700.669745:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878700.669748:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878700.669752:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878700.669756:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878700.669766:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511532032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878700.669775:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878700.669778:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878700.669780:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878700.669784:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3687 00000100:00080000:2.0:1788878700.669787:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878700.669791:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878700.669794:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878700.669798:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878700/real 1788878700] req@ffffa061888afb80 x1875771511531648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878700.669806:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511532160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878700.669826:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511531648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878700.669838:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878700.669842:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878700.669846:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511531648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878700.669868:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878700.669886:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878700.669895:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878700/real 1788878700] req@ffffa061888adc00 x1875771511531776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878700.669905:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511531776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878700.669915:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878700.669920:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878700.669925:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878700.669933:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878700.669957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878700.669972:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878700/real 1788878700] req@ffffa061888af480 x1875771511531904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878700.669983:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511531904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878700.669990:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878700.669993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878700.669996:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511531904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878700.670001:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878700.670017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878700.670025:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878700/real 1788878700] req@ffffa061888aca80 x1875771511532032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878700.670034:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511532032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878700.670041:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878700.670043:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878700.670045:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878700.670050:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878700.670062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878700.670070:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878700/real 1788878700] req@ffffa061888ac000 x1875771511532160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878700.670079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511532160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878700.670085:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878700.670087:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878700.670089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878701.309587:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878701.309711:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878701.565476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878702.333469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878702.333639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878702.589488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878703.357531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878703.357645:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878703.613550:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878704.381503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878704.381599:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878704.637538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878705.405494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878705.405657:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878705.661518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878705.789596:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878705.789610:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878705.789616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878705.789632:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3692 00000100:00080000:2.0:1788878705.789648:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878705.789661:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878705.789737:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511532288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878705.789790:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878705.789795:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878705.789797:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878705.789803:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3692 00000100:00080000:2.0:1788878705.789807:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878705.789813:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878705.789839:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878705.789861:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511532288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878705.789869:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511532416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878705.789897:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878705.789904:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878705.789908:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878705.789916:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3692 00000100:00080000:2.0:1788878705.789925:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878705.789951:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878705.789961:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878705.789988:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511532544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878705.790009:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878705.790017:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878705.790022:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878705.790031:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3692 00000100:00080000:2.0:1788878705.790040:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878705.790049:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532416:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878705.790057:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878705.790072:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878705.790101:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511532672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878705.790106:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532544:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878705.790136:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878705.790143:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878705.790147:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878705.790156:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3692 00000100:00080000:2.0:1788878705.790192:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878705.790194:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532672:192.168.203.111@tcp:8:kworker.0 00000100:00080000:2.0:1788878705.790205:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878705.790233:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878705.790255:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511532800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878705.790290:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878706.429485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878706.429604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878706.685502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878707.453462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878707.453537:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878707.709470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878708.477518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878708.477695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878708.733471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878708.861649:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878708.861694:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878708.861710:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878708.861748:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878708.861757:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878708.861767:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878708.861797:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878708.861825:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878705/real 1788878708] req@ffffa061888af480 x1875771511532800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878708.861854:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511532800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878708.861879:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878708.861884:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878708.861889:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878708.861899:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878705/real 1788878708] req@ffffa061888aca80 x1875771511532672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878708.861912:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511532672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878708.861921:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878708.861924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878708.861928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878708.861936:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878705/real 1788878708] req@ffffa061888adf80 x1875771511532416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878708.861959:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511532416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878708.861967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878708.861971:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878708.861975:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878708.861986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878705/real 1788878708] req@ffffa061888ac000 x1875771511532544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878708.861998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511532544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878708.862007:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878708.862011:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878708.862015:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511532544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878708.862022:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878705/real 1788878708] req@ffffa061888ad500 x1875771511532288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878708.862035:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511532288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878708.862043:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878708.862047:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878708.862051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511532288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878709.501542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878709.501625:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878709.757514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878710.525519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878710.525692:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878710.781543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878710.909611:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878710.909627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878710.909634:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878710.909650:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3697 00000100:00080000:3.0:1788878710.909667:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878710.909678:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878710.909733:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511532928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878710.909787:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878710.909793:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878710.909796:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878710.909801:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3697 00000100:00080000:3.0:1788878710.909807:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878710.909813:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878710.909828:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878710.909851:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511532928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878710.909852:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511533056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878710.909880:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878710.909884:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878710.909886:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878710.909890:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3697 00000100:00080000:3.0:1788878710.909894:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878710.909899:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878710.909907:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878710.909924:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511533184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878710.909937:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878710.909941:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878710.909944:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878710.909948:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3697 00000100:00080000:3.0:1788878710.909955:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878710.909959:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878710.909965:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878710.909978:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511533312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878710.910005:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878710.910009:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878710.910011:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878710.910014:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3697 00000100:00080000:3.0:1788878710.910018:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878710.910025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878710.910027:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878710.910034:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878710.910051:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511533440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878710.910091:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878710/real 1788878710] req@ffffa06188870a80 x1875771511532928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878710.910121:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511532928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878710.910160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878710.910167:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878710.910173:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511532928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878710.910211:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878710.910249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878710.910271:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878710/real 1788878710] req@ffffa06188870380 x1875771511533056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878710.910291:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511533056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878710.910314:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878710.910320:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878710.910326:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878710.910342:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878710.910376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878710.910429:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878710/real 1788878710] req@ffffa06188871c00 x1875771511533184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878710.910448:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511533184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878710.910460:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878710.910465:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878710.910470:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878710.910480:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878710.910512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878710.910529:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878710/real 1788878710] req@ffffa06188871880 x1875771511533312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878710.910546:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511533312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878710.910558:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878710.910563:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878710.910568:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878710.910576:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878710.910607:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878710.910624:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878710/real 1788878710] req@ffffa06188872d80 x1875771511533440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878710.910642:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511533440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878710.910653:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878710.910657:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878710.910662:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878711.549511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878711.549733:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878711.805672:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878712.573483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878712.573594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878712.829542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878713.597504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878713.597665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878713.853483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878714.621506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878714.621585:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878714.877528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878715.645475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878715.645588:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878715.901523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878716.029533:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878716.029545:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878716.029552:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878716.029567:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3702 00000100:00080000:3.0:1788878716.029582:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878716.029594:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878716.029642:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511533568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878716.029672:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878716.029676:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878716.029678:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878716.029682:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3702 00000100:00080000:3.0:1788878716.029686:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878716.029690:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878716.029701:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878716.029722:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511533696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878716.029724:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511533568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878716.029749:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878716.029753:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878716.029754:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878716.029758:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3702 00000100:00080000:3.0:1788878716.029761:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878716.029767:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878716.029771:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878716.029783:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511533824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878716.029798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878716.029801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878716.029803:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878716.029808:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3702 00000100:00080000:3.0:1788878716.029811:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878716.029814:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878716.029819:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878716.029822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878716.029837:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511533952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878716.029850:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878716.029853:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878716.029856:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878716.029862:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3702 00000100:00080000:3.0:1788878716.029869:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878716.029875:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878716.029880:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878716.029881:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878716/real 1788878716] req@ffffa06188870700 x1875771511533568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878716.029895:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511534080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878716.029906:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511533568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878716.029920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878716.029927:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878716.029933:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511533568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878716.029963:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878716.029994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878716.030005:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878716/real 1788878716] req@ffffa06188873b80 x1875771511533696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878716.030015:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511533696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878716.030023:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878716.030026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878716.030029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878716.030034:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878716.030048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878716.030056:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878716/real 1788878716] req@ffffa06188872300 x1875771511533824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878716.030065:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511533824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878716.030071:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878716.030073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878716.030076:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878716.030081:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878716.030109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878716.030119:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878716/real 1788878716] req@ffffa06188873800 x1875771511533952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878716.030128:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511533952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878716.030134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878716.030137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878716.030139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511533952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878716.030144:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878716.030157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878716.030164:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878716/real 1788878716] req@ffffa06188872680 x1875771511534080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878716.030173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511534080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878716.030180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878716.030182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878716.030185:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878716.669500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878716.669603:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878716.925520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878717.693498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878717.693570:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878717.949456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878718.717439:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878718.717626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878718.973519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878719.741479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878719.741538:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878719.997519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878720.765488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878720.765563:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878721.021471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878721.149496:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878721.149511:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878721.149518:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878721.149534:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3707 00000100:00080000:3.0:1788878721.149551:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878721.149562:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878721.149617:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511534208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878721.149649:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878721.149654:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878721.149656:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878721.149661:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3707 00000100:00080000:3.0:1788878721.149665:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878721.149670:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878721.149682:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878721.149694:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511534208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878721.149710:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511534336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878721.149725:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878721.149729:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878721.149731:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878721.149735:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3707 00000100:00080000:3.0:1788878721.149739:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878721.149743:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878721.149749:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878721.149761:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511534464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878721.149771:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878721.149777:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878721.149779:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878721.149783:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3707 00000100:00080000:3.0:1788878721.149787:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878721.149790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878721.149796:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878721.149805:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878721.149818:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511534592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878721.149830:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878721.149834:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878721.149836:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878721.149845:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3707 00000100:00100000:0.0:1788878721.149846:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878721/real 1788878721] req@ffffa06188873100 x1875771511534208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878721.149848:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878721.149858:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878721.149868:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878721.149896:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511534208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878721.149896:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511534720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878721.149948:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878721.149956:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878721.149961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511534208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878721.149988:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878721.150012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878721.150026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878721/real 1788878721] req@ffffa06188871500 x1875771511534336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878721.150047:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511534336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878721.150057:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878721.150084:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878721.150089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878721.150127:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878721.150146:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878721.150156:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878721/real 1788878721] req@ffffa06188873480 x1875771511534464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878721.150166:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511534464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878721.150173:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878721.150176:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878721.150179:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878721.150184:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878721.150196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878721.150203:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878721/real 1788878721] req@ffffa06188872a00 x1875771511534592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878721.150213:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511534592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878721.150225:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878721.150228:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878721.150230:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878721.150234:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878721.150245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878721.150252:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878721/real 1788878721] req@ffffa06188872680 x1875771511534720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878721.150261:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511534720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878721.150270:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878721.150275:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878721.150279:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878721.789548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878721.789666:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878722.045546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878722.813468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878722.813529:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878723.069468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878723.837480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878723.837571:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878724.093551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878724.861518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878724.861591:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878725.117522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878725.885528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878725.885660:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878726.141502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878726.269552:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878726.269566:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878726.269573:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878726.269591:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3712 00000100:00080000:3.0:1788878726.269607:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878726.269618:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878726.269700:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511534848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878726.269755:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878726.269761:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878726.269763:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878726.269769:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3712 00000100:00080000:3.0:1788878726.269773:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878726.269779:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878726.269794:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878726.269812:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511534976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878726.269827:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878726.269834:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511534848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878726.269834:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878726.269837:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878726.269850:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3712 00000100:00080000:3.0:1788878726.269857:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878726.269862:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878726.269868:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878726.269888:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511535104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878726.269900:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878726.269905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878726.269908:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878726.269914:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3712 00000100:00080000:3.0:1788878726.269920:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878726.269925:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878726.269932:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878726.269970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878726.269971:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511535232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878726.269988:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878726.269992:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878726.269995:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878726.270000:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3712 00000100:00080000:3.0:1788878726.270005:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878726.270010:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878726.270017:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878726.270063:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878726/real 1788878726] req@ffffa06188873800 x1875771511534848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878726.270068:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511535360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878726.270146:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511534848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878726.270160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878726.270166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878726.270171:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511534848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878726.270202:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878726.270234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878726.270251:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878726/real 1788878726] req@ffffa06188872300 x1875771511534976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878726.270265:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511534976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878726.270276:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878726.270279:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878726.270283:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511534976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878726.270290:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878726.270308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878726.270319:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878726/real 1788878726] req@ffffa06188873b80 x1875771511535104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878726.270332:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511535104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878726.270349:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878726.270373:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878726.270377:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878726.270384:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878726.270444:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878726.270460:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878726/real 1788878726] req@ffffa06188870700 x1875771511535232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878726.270475:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511535232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878726.270486:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878726.270490:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878726.270493:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878726.270500:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878726.270520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878726.270531:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878726/real 1788878726] req@ffffa06188872d80 x1875771511535360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878726.270545:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511535360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878726.270555:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878726.270558:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878726.270563:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878726.909474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878726.909552:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878727.165519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878727.933485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878727.933599:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878728.189523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878728.957484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878728.957544:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878729.213509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878729.981518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878729.981617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878730.237515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878731.005490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878731.005590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878731.261457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878731.389505:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878731.389537:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878731.389543:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878731.389560:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3718 00000100:00080000:3.0:1788878731.389577:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878731.389590:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878731.389644:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511535488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878731.389700:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878731.389710:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878731.389714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878731.389722:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3718 00000100:00080000:3.0:1788878731.389730:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878731.389737:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878731.389746:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511535488:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878731.389755:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878731.389783:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511535616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878731.389801:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878731.389806:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878731.389809:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878731.389817:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3718 00000100:00080000:3.0:1788878731.389824:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878731.389826:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878731.389837:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878731.389847:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878731.389869:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511535744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878731.389885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878731/real 1788878731] req@ffffa06188871880 x1875771511535488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878731.389887:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878731.389901:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878731.389908:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878731.389922:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3718 00000100:00080000:3.0:1788878731.389929:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878731.389932:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511535488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878731.389946:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878731.389955:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878731.389956:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878731.389961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511535488:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878731.389972:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878731.390012:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878731.390017:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511535872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878731.390038:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878731.390044:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878731.390047:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878731.390052:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3718 00000800:00000100:0.0:1788878731.390060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878731.390061:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878731.390074:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878731.390082:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878731.390088:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878731/real 1788878731] req@ffffa06188871c00 x1875771511535616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878731.390108:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511536000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878731.390111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511535616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878731.390151:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878731.390158:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878731.390165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878731.390177:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878731.390204:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878731.390222:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878731/real 1788878731] req@ffffa06188870380 x1875771511535744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878731.390242:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511535744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878731.390255:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878731.390260:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878731.390265:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878731.390284:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878731.390307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878731.390318:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878731/real 1788878731] req@ffffa06188870a80 x1875771511535872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878731.390329:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511535872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878731.390337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878731.390340:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878731.390343:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511535872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878731.390348:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878731.390363:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878731.390371:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878731/real 1788878731] req@ffffa06188871880 x1875771511536000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878731.390407:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511536000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878731.390415:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878731.390418:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878731.390421:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878732.029475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878732.029620:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878732.285519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878733.053559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878733.053661:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878733.309584:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878734.077540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878734.077679:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878734.333518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878735.101515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878735.101632:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878735.357492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878736.125486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878736.125652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878736.381508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878736.509574:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878736.509598:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878736.509603:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878736.509617:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3723 00000100:00080000:2.0:1788878736.509631:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878736.509641:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878736.509688:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511536128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878736.509722:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878736.509730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878736.509733:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878736.509738:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3723 00000100:00080000:2.0:1788878736.509743:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878736.509748:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878736.509761:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878736.509779:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511536128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878736.509781:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511536256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878736.509805:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878736.509810:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878736.509812:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878736.509817:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3723 00000100:00080000:2.0:1788878736.509821:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878736.509825:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878736.509831:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878736.509846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511536384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878736.509857:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878736.509862:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878736.509865:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878736.509869:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3723 00000100:00080000:2.0:1788878736.509891:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878736.509899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878736.509903:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878736.509914:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878736.509932:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511536512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878736.509946:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878736.509950:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878736.509974:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878736.509984:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878736/real 1788878736] req@ffffa061888adc00 x1875771511536128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878736.510011:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3723 00000100:00080000:2.0:1788878736.510022:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878736.510032:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878736.510040:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878736.510053:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511536128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878736.510061:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511536640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878736.510104:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878736.510114:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878736.510121:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511536128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878736.510158:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878736.510224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878736.510251:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878736/real 1788878736] req@ffffa061888afb80 x1875771511536256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878736.510277:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511536256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878736.510289:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878736.510294:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878736.510298:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878736.510307:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878736.510335:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878736.510349:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878736/real 1788878736] req@ffffa061888aea00 x1875771511536384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878736.510363:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511536384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878736.510390:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878736.510394:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878736.510397:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878736.510404:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878736.510425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878736.510490:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878736/real 1788878736] req@ffffa061888ae300 x1875771511536512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878736.510506:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511536512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878736.510517:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878736.510521:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878736.510525:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878736.510532:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878736.510574:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878736.510586:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878736/real 1788878736] req@ffffa061888af100 x1875771511536640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878736.510599:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511536640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878736.510608:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878736.510611:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878736.510614:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878737.149540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878737.149655:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878737.405525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878738.173482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878738.173664:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878738.429512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878739.197466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878739.197520:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878739.453477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878740.221500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878740.221673:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878740.477541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878741.245511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878741.245699:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878741.501557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878741.629548:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878741.629569:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878741.629574:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878741.629587:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3728 00000100:00080000:3.0:1788878741.629600:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878741.629609:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878741.629675:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511536768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878741.629724:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878741.629730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878741.629733:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878741.629738:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3728 00000100:00080000:3.0:1788878741.629744:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878741.629750:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878741.629765:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878741.629784:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511536896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878741.629793:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511536768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878741.629809:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878741.629816:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878741.629818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878741.629824:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3728 00000100:00080000:3.0:1788878741.629828:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878741.629834:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878741.629840:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878741.629856:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511537024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878741.629869:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878741.629874:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878741.629876:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878741.629881:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3728 00000100:00080000:3.0:1788878741.629887:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878741.629892:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878741.629899:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878741.629907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878741.629968:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511537152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878741.629994:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878741.629997:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878741.629999:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878741.630003:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3728 00000100:00080000:3.0:1788878741.630008:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878741.630012:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878741.630017:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878741.630020:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878741/real 1788878741] req@ffffa06188872d80 x1875771511536768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878741.630031:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511537280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878741.630069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511536768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878741.630082:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878741.630086:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878741.630090:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511536768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878741.630114:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878741.630138:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878741.630150:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878741/real 1788878741] req@ffffa06188870700 x1875771511536896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878741.630161:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511536896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878741.630168:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878741.630171:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878741.630174:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511536896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878741.630178:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878741.630191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878741.630199:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878741/real 1788878741] req@ffffa06188873b80 x1875771511537024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878741.630214:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511537024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878741.630221:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878741.630223:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878741.630225:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878741.630230:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878741.630243:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878741.630251:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878741/real 1788878741] req@ffffa06188872300 x1875771511537152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878741.630259:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511537152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878741.630266:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878741.630268:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878741.630271:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878741.630276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878741.630287:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878741.630294:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878741/real 1788878741] req@ffffa06188873800 x1875771511537280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878741.630302:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511537280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878741.630315:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878741.630328:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878741.630331:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878742.269553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878742.269658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878742.525789:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878743.293502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878743.293594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878743.549533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878744.317459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878744.317551:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878744.573502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878745.341527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878745.341652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878745.597545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878746.365511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878746.365610:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878746.621632:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878746.749556:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878746.749571:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878746.749577:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878746.749595:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3733 00000100:00080000:3.0:1788878746.749611:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878746.749623:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878746.749703:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511537408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878746.749743:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878746.749749:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878746.749751:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878746.749757:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3733 00000100:00080000:3.0:1788878746.749762:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878746.749766:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878746.749781:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878746.749799:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511537536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878746.749815:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878746.749820:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878746.749823:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878746.749833:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511537408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878746.749833:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3733 00000100:00080000:3.0:1788878746.749839:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878746.749852:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878746.749866:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878746.749935:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511537664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878746.749953:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878746.749960:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878746.749963:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878746.749970:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3733 00000100:00080000:3.0:1788878746.749977:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878746.749984:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878746.749993:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878746.750003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878746.750031:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511537792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878746.750047:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878746.750052:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878746.750056:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878746.750062:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3733 00000100:00080000:3.0:1788878746.750069:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878746.750075:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878746.750082:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878746.750096:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511537920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878746.750109:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878746/real 1788878746] req@ffffa06188872680 x1875771511537408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878746.750164:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511537408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878746.750181:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878746.750188:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878746.750194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511537408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878746.750232:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878746.750280:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878746.750299:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878746/real 1788878746] req@ffffa06188872a00 x1875771511537536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878746.750328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511537536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878746.750342:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878746.750348:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878746.750354:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878746.750381:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878746.750461:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878746.750487:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878746/real 1788878746] req@ffffa06188873480 x1875771511537664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878746.750512:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511537664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878746.750528:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878746.750535:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878746.750542:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878746.750579:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878746.750613:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878746.750630:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878746/real 1788878746] req@ffffa06188871500 x1875771511537792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878746.750668:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511537792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878746.750697:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878746.750704:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878746.750710:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878746.750721:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878746.750758:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878746.750779:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878746/real 1788878746] req@ffffa06188873100 x1875771511537920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878746.750804:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511537920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878746.750821:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878746.750828:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878746.750835:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511537920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878747.389539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878747.389672:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878747.645553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878748.413449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878748.413563:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878748.669594:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878749.437495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878749.437611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878749.693548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878750.461510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878750.461650:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878750.717562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878751.485499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878751.485623:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878751.741534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878751.869553:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878751.869565:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878751.869572:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878751.869588:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3738 00000100:00080000:2.0:1788878751.869603:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878751.869615:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878751.869666:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511538048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878751.869705:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878751.869710:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878751.869713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878751.869718:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3738 00000100:00080000:2.0:1788878751.869723:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878751.869729:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878751.869743:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878751.869764:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511538048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878751.869765:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511538176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878751.869792:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878751.869797:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878751.869799:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878751.869803:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3738 00000100:00080000:2.0:1788878751.869808:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878751.869812:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878751.869818:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878751.869835:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511538304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878751.869877:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878751.869884:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878751.869887:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878751.869893:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3738 00000100:00080000:2.0:1788878751.869899:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878751.869905:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878751.869913:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878751.869919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878751.869938:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511538432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878751.869954:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878751.869958:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878751.869960:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878751.869967:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3738 00000100:00080000:2.0:1788878751.869973:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878751.869983:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878751.869985:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878751/real 1788878751] req@ffffa061888ae680 x1875771511538048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878751.869991:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878751.870013:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511538560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878751.870068:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511538048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878751.870087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878751.870094:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878751.870099:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511538048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878751.870130:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878751.870163:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878751.870196:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878751/real 1788878751] req@ffffa061888ac380 x1875771511538176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878751.870212:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511538176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878751.870222:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878751.870226:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878751.870231:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878751.870238:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878751.870260:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878751.870272:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878751/real 1788878751] req@ffffa061888ad880 x1875771511538304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878751.870286:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511538304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878751.870304:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878751.870308:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878751.870312:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878751.870318:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878751.870338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878751.870349:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878751/real 1788878751] req@ffffa061888aed80 x1875771511538432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878751.870363:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511538432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878751.870373:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878751.870376:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878751.870411:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878751.870420:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878751.870443:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878751.870457:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878751/real 1788878751] req@ffffa061888ac700 x1875771511538560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878751.870472:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511538560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878751.870482:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878751.870485:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878751.870490:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878752.509546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878752.509653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878752.765481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878753.533520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878753.533695:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878753.789547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878754.557532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878754.557630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878754.813481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878755.581509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878755.581629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878755.837647:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878756.605508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878756.605695:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878756.861627:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878756.989569:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878756.989582:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878756.989588:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878756.989601:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3743 00000100:00080000:0.0:1788878756.989616:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878756.989625:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878756.989697:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511538688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878756.989748:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878756.989755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878756.989758:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878756.989764:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3743 00000100:00080000:0.0:1788878756.989769:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878756.989774:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878756.989786:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878756.989803:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511538816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878756.989892:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878756.989897:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878756.989901:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878756.989909:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3743 00000100:00080000:0.0:1788878756.989916:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878756.989924:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878756.989949:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878756.989963:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511538688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878756.989978:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511538944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878756.989996:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878756.990001:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878756.990004:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878756.990009:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3743 00000100:00080000:0.0:1788878756.990033:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878756.990038:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878756.990045:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878756.990058:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511539072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878756.990069:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878756.990073:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878756.990079:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788878756.990086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878756.990088:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3743 00000100:00080000:0.0:1788878756.990098:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878756.990102:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878756.990107:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878756.990118:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511539200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878756.990166:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878756/real 1788878756] req@ffffa06198afdf80 x1875771511538688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878756.990223:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511538688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878756.990244:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878756.990252:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878756.990258:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511538688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878756.990270:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878756.990309:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878756.990330:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878756/real 1788878756] req@ffffa06198afdc00 x1875771511538816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878756.990353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511538816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878756.990372:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878756.990377:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878756.990382:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878756.990500:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878756.990534:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878756.990551:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878756/real 1788878756] req@ffffa06198afc700 x1875771511538944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878756.990568:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511538944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878756.990579:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878756.990584:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878756.990588:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511538944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878756.990598:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878756.990629:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878756.990644:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878756/real 1788878756] req@ffffa06198afc000 x1875771511539072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878756.990666:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511539072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878756.990674:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878756.990677:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878756.990680:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878756.990687:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878756.990704:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878756.990724:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878756/real 1788878756] req@ffffa06198afea00 x1875771511539200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878756.990734:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511539200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878756.990741:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878756.990744:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878756.990746:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878757.629512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878757.629621:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878757.885484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878758.653537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878758.653621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878758.909555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878759.677590:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878759.677664:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878759.933488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878760.701527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878760.701675:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878760.957502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878761.725522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878761.725840:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878761.981467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878762.109581:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878762.109596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878762.109602:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878762.109615:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3748 00000100:00080000:3.0:1788878762.109629:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878762.109638:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878762.109689:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511539328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878762.109721:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878762.109725:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878762.109728:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878762.109733:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3748 00000100:00080000:3.0:1788878762.109737:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878762.109744:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878762.109756:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878762.109757:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511539328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878762.109789:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511539456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878762.109838:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878762.109844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878762.109845:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878762.109849:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878762.109853:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3748 00000100:00080000:3.0:1788878762.109859:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878762.109863:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878762.109869:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878762.109883:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511539584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878762.109886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878762/real 1788878762] req@ffffa06188873100 x1875771511539328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878762.109907:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878762.109911:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878762.109912:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878762.109916:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3748 00000100:00080000:3.0:1788878762.109918:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878762.109921:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878762.109925:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511539328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878762.109926:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878762.109938:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511539712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878762.109940:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878762.109948:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788878762.109953:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878762.109955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511539328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878762.109960:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878762.109962:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878762.109966:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3748 00000100:00080000:3.0:1788878762.109968:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878762.109970:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878762.109973:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878762.109981:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511539840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878762.109983:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878762.110009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878762.110020:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878762/real 1788878762] req@ffffa06188871500 x1875771511539456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878762.110034:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511539456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878762.110041:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878762.110043:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878762.110045:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878762.110049:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878762.110060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878762.110066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878762/real 1788878762] req@ffffa06188873480 x1875771511539584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878762.110074:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511539584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878762.110079:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878762.110081:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878762.110082:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878762.110086:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878762.110096:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878762.110102:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878762/real 1788878762] req@ffffa06188872a00 x1875771511539712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878762.110109:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511539712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878762.110119:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878762.110121:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878762.110123:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878762.110133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878762.110142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878762.110148:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878762/real 1788878762] req@ffffa06188872680 x1875771511539840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878762.110155:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511539840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878762.110161:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878762.110162:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878762.110164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511539840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878762.749509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878762.749876:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878763.005523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878763.773456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878763.773542:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878764.029515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878764.797523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878764.797621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878765.053508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878765.821480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878765.821531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878766.077500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878766.845508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878766.845570:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878767.101523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878767.229511:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878767.229524:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878767.229529:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878767.229540:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3753 00000100:00080000:3.0:1788878767.229551:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878767.229560:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878767.229602:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511539968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878767.229639:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878767.229643:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878767.229646:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878767.229650:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3753 00000100:00080000:3.0:1788878767.229655:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878767.229659:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878767.229670:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878767.229685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511540096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878767.229699:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878767.229705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878767.229708:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878767.229713:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3753 00000100:00080000:3.0:1788878767.229717:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878767.229722:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878767.229731:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878767.229733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511539968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878767.229765:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511540224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878767.229792:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878767.229796:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878767.229799:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878767.229803:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3753 00000100:00080000:3.0:1788878767.229808:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878767.229814:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878767.229821:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878767.229838:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511540352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878767.229850:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878767.229854:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878767.229856:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878767.229865:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3753 00000800:00000100:0.0:1788878767.229868:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878767.229874:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878767.229883:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878767.229891:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878767.229911:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511540480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878767.229956:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878767/real 1788878767] req@ffffa06188873800 x1875771511539968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878767.230014:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511539968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878767.230024:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878767.230027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878767.230030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511539968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878767.230037:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878767.230083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878767.230098:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878767/real 1788878767] req@ffffa06188872300 x1875771511540096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878767.230111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511540096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878767.230122:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878767.230125:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878767.230129:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878767.230160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878767.230178:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878767.230189:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878767/real 1788878767] req@ffffa06188873b80 x1875771511540224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878767.230201:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511540224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878767.230217:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878767.230220:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878767.230224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878767.230254:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878767.230267:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878767.230273:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878767/real 1788878767] req@ffffa06188870700 x1875771511540352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878767.230281:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511540352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878767.230286:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878767.230288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878767.230290:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878767.230294:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878767.230303:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878767.230309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878767/real 1788878767] req@ffffa06188872d80 x1875771511540480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878767.230315:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511540480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878767.230328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878767.230330:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878767.230333:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878767.869449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878767.869592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878768.125486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878768.893497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878768.893655:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878769.149533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878769.917562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878769.917767:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878770.173590:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878770.941558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878770.941665:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878771.197477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878771.965464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878771.965526:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878772.221489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878772.349497:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878772.349508:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878772.349513:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878772.349525:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3759 00000100:00080000:3.0:1788878772.349539:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878772.349549:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878772.349599:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511540608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878772.349625:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878772.349630:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878772.349632:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878772.349637:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3759 00000100:00080000:3.0:1788878772.349642:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878772.349648:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878772.349659:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511540608:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878772.349660:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878772.349679:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511540736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878772.349688:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878772.349694:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878772.349695:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878772.349698:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3759 00000100:00080000:3.0:1788878772.349701:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878772.349704:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878772.349721:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878772.349730:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511540864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878772.349739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878772.349742:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878772.349743:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878772.349747:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3759 00000100:00080000:3.0:1788878772.349750:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878772.349753:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878772.349757:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878772.349765:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511540992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878772.349773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878772.349776:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540736:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788878772.349778:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878772.349783:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878772.349789:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3759 00000100:00080000:3.0:1788878772.349794:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878772.349801:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878772.349808:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878772.349809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878772.349833:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511541120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878772.349839:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878772.349880:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878772.989453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878772.989521:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878773.245499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878774.013510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878774.013645:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878774.269484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878775.037464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878775.037512:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878775.293500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788878775.421696:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788878775.421740:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788878775.421756:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788878775.421795:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788878775.421804:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788878775.421817:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788878775.421828:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878775.421848:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878772/real 1788878775] req@ffffa06188871c00 x1875771511540992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878775.421877:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511540992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878775.421889:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.421895:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878775.421900:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878775.421908:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878772/real 1788878775] req@ffffa06188870a80 x1875771511540736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878775.421921:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511540736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878775.421930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.421934:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878775.421938:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878775.421945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878772/real 1788878775] req@ffffa06188870380 x1875771511540864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878775.421957:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511540864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878775.421964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.421968:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878775.421970:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511540864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878775.421977:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878772/real 1788878775] req@ffffa06188871880 x1875771511540608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878775.421987:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511540608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878775.421995:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.421998:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878775.422002:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511540608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878775.422030:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878772/real 1788878775] req@ffffa06188872d80 x1875771511541120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878775.422044:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511541120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878775.422052:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.422055:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878775.422058:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878776.061460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878776.061516:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878776.317456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878777.085516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878777.085580:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878777.341500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878777.469555:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878777.469568:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878777.469573:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878777.469587:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3764 00000100:00080000:0.0:1788878777.469613:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878777.469625:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878777.469717:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511541248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878777.469766:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878777.469772:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878777.469774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878777.469790:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3764 00000100:00080000:0.0:1788878777.469797:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878777.469804:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878777.469820:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878777.469846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511541376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878777.469862:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511541248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878777.469889:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878777.469897:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878777.469902:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878777.469910:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3764 00000100:00080000:0.0:1788878777.469918:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878777.469925:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878777.469937:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878777.469961:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511541504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878777.469994:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788878777.470003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878777.470006:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878777.470013:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878777.470022:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3764 00000100:00080000:0.0:1788878777.470030:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878777.470037:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878777.470045:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878777.470062:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511541632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878777.470071:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878777/real 1788878777] req@ffffa06198afe300 x1875771511541248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878777.470078:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878777.470088:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878777.470094:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878777.470107:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3764 00000100:00080000:0.0:1788878777.470112:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878777.470116:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878777.470122:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878777.470130:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511541248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878777.470136:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511541760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878777.470178:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878777.470188:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878777.470197:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511541248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878777.470243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878777.470289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878777.470312:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878777/real 1788878777] req@ffffa06198afc380 x1875771511541376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878777.470333:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511541376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878777.470348:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878777.470354:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878777.470360:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878777.470370:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878777.470441:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878777.470461:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878777/real 1788878777] req@ffffa06198afca80 x1875771511541504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878777.470481:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511541504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878777.470494:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878777.470499:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878777.470505:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878777.470515:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878777.470550:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878777.470570:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878777/real 1788878777] req@ffffa06198afd500 x1875771511541632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878777.470587:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511541632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878777.470610:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878777.470616:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878777.470621:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878777.470630:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878777.470664:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878777.470708:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878777/real 1788878777] req@ffffa06198afd180 x1875771511541760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878777.471743:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511541760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878777.471758:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878777.471764:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878777.471770:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511541760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878778.109492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878778.109662:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878778.365536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878779.133517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878779.133723:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878779.389560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878780.157570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878780.157617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878780.413549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878781.181500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878781.181575:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878781.437528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878782.205515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878782.205584:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878782.461513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878782.589513:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878782.589524:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878782.589529:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878782.589542:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3769 00000100:00080000:3.0:1788878782.589555:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878782.589563:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878782.589610:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511541888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878782.589651:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878782.589655:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878782.589657:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878782.589662:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3769 00000100:00080000:3.0:1788878782.589666:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878782.589669:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878782.589680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878782.589681:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511541888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878782.589694:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511542016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878782.589709:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878782.589711:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878782.589713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878782.589715:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3769 00000100:00080000:3.0:1788878782.589718:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878782.589720:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878782.589724:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878782.589732:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511542144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878782.589739:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878782.589742:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878782.589743:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878782.589745:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3769 00000100:00080000:3.0:1788878782.589747:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878782.589750:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878782.589754:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878782.589760:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511542272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878782.589769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878782.589770:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878782.589775:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878782.589776:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878782.589779:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3769 00000100:00080000:3.0:1788878782.589783:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878782.589786:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878782.589789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878782.589796:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511542400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878782.589820:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878782/real 1788878782] req@ffffa06188870700 x1875771511541888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878782.589838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511541888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878782.589847:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878782.589850:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878782.589853:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511541888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878782.589876:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878782.589896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878782.589904:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878782/real 1788878782] req@ffffa06188873b80 x1875771511542016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878782.589913:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511542016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878782.589919:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878782.589922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878782.589924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878782.589927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878782.589938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878782.589944:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878782/real 1788878782] req@ffffa06188872300 x1875771511542144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878782.589951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511542144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878782.589957:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878782.589959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878782.589961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878782.589965:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878782.589977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878782.589982:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878782/real 1788878782] req@ffffa06188873800 x1875771511542272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878782.589995:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511542272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878782.590001:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878782.590002:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878782.590004:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878782.590007:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878782.590018:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878782.590024:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878782/real 1788878782] req@ffffa06188872680 x1875771511542400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878782.590032:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511542400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878782.590038:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878782.590039:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878782.590041:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878783.229546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878783.229669:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878783.485529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878784.253527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878784.253559:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878784.509538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878785.277466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878785.277680:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878785.533531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878786.301538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878786.301654:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878786.557589:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878787.325544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878787.325689:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878787.581513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878787.709539:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878787.709553:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878787.709575:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878787.709593:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3774 00000100:00080000:0.0:1788878787.709640:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878787.709655:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878787.709721:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511542528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878787.709763:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878787.709768:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878787.709771:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878787.709776:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3774 00000100:00080000:0.0:1788878787.709782:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878787.709789:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878787.709804:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878787.709822:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511542656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878787.709837:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878787.709842:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511542528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878787.709843:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878787.709861:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878787.709868:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3774 00000100:00080000:0.0:1788878787.709872:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878787.709877:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878787.709885:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878787.709903:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511542784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878787.709926:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878787.709931:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878787.709934:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878787.709938:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3774 00000100:00080000:0.0:1788878787.709943:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878787.709952:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788878787.709953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:0.0:1788878787.709965:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878787.709984:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511542912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878787.709996:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878787.710001:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878787.710004:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878787.710011:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3774 00000100:00100000:3.0:1788878787.710018:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878787/real 1788878787] req@ffffa06198afd180 x1875771511542528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878787.710020:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878787.710036:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878787.710051:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878787.710075:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511542528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878787.710076:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511543040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878787.710145:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878787.710155:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878787.710162:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511542528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878787.710205:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878787.710248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878787.710289:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878787/real 1788878787] req@ffffa06198afd500 x1875771511542656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878787.710308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511542656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878787.710319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878787.710324:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878787.710330:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878787.710338:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878787.710360:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878787.710373:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878787/real 1788878787] req@ffffa06198afca80 x1875771511542784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878787.710420:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511542784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878787.710430:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878787.710434:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878787.710438:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878787.710446:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878787.710470:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878787.710484:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878787/real 1788878787] req@ffffa06198afc380 x1875771511542912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878787.710498:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511542912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878787.710508:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878787.710512:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878787.710516:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511542912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878787.710522:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878787.710541:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878787.710553:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878787/real 1788878787] req@ffffa06198afe300 x1875771511543040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878787.710567:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511543040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878787.710576:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878787.710580:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878787.710583:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878788.349519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878788.349704:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878788.605674:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878789.373468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878789.373696:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878789.629558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878790.397601:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878790.397704:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878790.653547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878791.421548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878791.421687:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878791.677527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878792.445522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878792.445608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878792.701494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878792.829510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878792.829520:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878792.829524:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878792.829536:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3779 00000100:00080000:0.0:1788878792.829548:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878792.829556:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878792.829632:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511543168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878792.829662:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878792.829665:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878792.829667:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878792.829671:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3779 00000100:00080000:0.0:1788878792.829674:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878792.829677:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878792.829688:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878792.829700:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511543296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878792.829711:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878792.829714:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878792.829715:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878792.829720:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3779 00000100:00080000:0.0:1788878792.829725:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878792.829725:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511543168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878792.829734:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878792.829740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878792.829752:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511543424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878792.829761:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878792.829764:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878792.829766:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878792.829769:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3779 00000100:00080000:0.0:1788878792.829772:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878792.829776:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878792.829780:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878792.829789:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511543552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878792.829798:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878792.829801:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878792.829803:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788878792.829807:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878792.829821:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3779 00000100:00080000:0.0:1788878792.829826:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878792.829829:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878792.829834:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878792.829846:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511543680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878792.829854:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878792/real 1788878792] req@ffffa06198afea00 x1875771511543168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878792.829888:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511543168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878792.829906:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878792.829914:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878792.829922:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511543168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878792.829957:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878792.829983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878792.829996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878792/real 1788878792] req@ffffa06198afc000 x1875771511543296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878792.830007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511543296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878792.830020:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878792.830023:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878792.830027:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878792.830032:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878792.830046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878792.830054:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878792/real 1788878792] req@ffffa06198afc700 x1875771511543424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878792.830063:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511543424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878792.830070:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878792.830073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878792.830075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878792.830080:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878792.830093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878792.830101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878792/real 1788878792] req@ffffa06198afdc00 x1875771511543552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878792.830111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511543552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878792.830117:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878792.830119:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878792.830123:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878792.830128:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878792.830140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878792.830147:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878792/real 1788878792] req@ffffa06198afdf80 x1875771511543680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878792.830156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511543680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878792.830163:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878792.830165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878792.830168:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878793.469511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878793.469718:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878793.725675:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878794.493512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878794.493641:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878794.749502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878795.517485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878795.517612:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878795.773579:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878796.541483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878796.541577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878796.797528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788878797.565502:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788878797.566431:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788878797.821482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878797.949599:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878797.949610:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878797.949616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878797.949632:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3784 00000100:00080000:0.0:1788878797.949648:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878797.949660:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878797.949717:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511543808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878797.949754:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878797.949759:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878797.949761:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878797.949766:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3784 00000100:00080000:0.0:1788878797.949771:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878797.949776:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878797.949787:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878797.949802:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511543936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878797.949816:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878797.949820:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878797.949823:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878797.949827:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3784 00000100:00080000:0.0:1788878797.949831:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878797.949835:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878797.949841:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878797.949858:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511544064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878797.949860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511543808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878797.949887:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878797.949891:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878797.949893:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878797.949897:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3784 00000100:00080000:0.0:1788878797.949901:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878797.949905:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878797.949910:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878797.949924:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511544192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878797.949943:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878797.949948:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878797.949950:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878797.949953:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3784 00000100:00080000:0.0:1788878797.949958:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878797.949962:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878797.949967:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878797.949998:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511544320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878797.950006:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878797.950089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878797/real 1788878797] req@ffffa06198aff800 x1875771511543808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878797.950133:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511543808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878797.950166:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878797.950172:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878797.950178:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511543808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878797.950191:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878797.950223:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878797.950239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878797/real 1788878797] req@ffffa06198affb80 x1875771511543936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878797.950252:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511543936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878797.950280:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878797.950283:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878797.950288:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511543936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878797.950330:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878797.950351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878797.950363:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878797/real 1788878797] req@ffffa06198afd880 x1875771511544064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878797.950375:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511544064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878797.950384:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878797.950387:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878797.950391:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878797.950397:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878797.950414:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878797.950487:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878797/real 1788878797] req@ffffa06198aff480 x1875771511544192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878797.950504:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511544192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878797.950515:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878797.950518:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878797.950522:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878797.950577:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878797.950606:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878797.950619:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878797/real 1788878797] req@ffffa06198afed80 x1875771511544320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878797.950644:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511544320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878797.950653:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878797.950657:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878797.950660:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878798.589489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878798.589584:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878798.845493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878799.613584:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878799.613665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878799.869553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878800.637575:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878800.637727:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878800.893501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878801.661461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878801.661553:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878801.917553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878802.685460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878802.685595:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878802.941477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878803.069606:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878803.069620:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878803.069625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878803.069639:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3789 00000100:00080000:0.0:1788878803.069670:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878803.069679:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878803.069774:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511544448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878803.069835:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878803.069860:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878803.069864:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878803.069873:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3789 00000100:00080000:0.0:1788878803.069882:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878803.069900:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878803.069919:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878803.069946:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511544576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878803.069948:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511544448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878803.069969:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878803.069975:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878803.069978:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878803.069985:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3789 00000100:00080000:0.0:1788878803.069991:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878803.069999:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878803.070007:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878803.070039:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511544704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878803.070051:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878803.070055:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878803.070057:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878803.070062:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3789 00000100:00080000:0.0:1788878803.070066:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878803.070072:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788878803.070076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:0.0:1788878803.070079:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878803.070092:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511544832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878803.070105:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878803.070109:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878803.070112:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878803.070128:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3789 00000100:00080000:0.0:1788878803.070133:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878803.070136:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878803/real 1788878803] req@ffffa06198afed80 x1875771511544448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878803.070139:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878803.070146:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878803.070159:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511544960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878803.070162:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511544448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878803.070177:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878803.070183:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878803.070189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511544448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878803.070222:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878803.070271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878803.070292:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878803/real 1788878803] req@ffffa06198aff480 x1875771511544576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878803.070335:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511544576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878803.070349:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878803.070353:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878803.070368:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878803.070377:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878803.070443:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878803.070459:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878803/real 1788878803] req@ffffa06198afd880 x1875771511544704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878803.070485:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511544704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878803.070515:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878803.070536:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878803.070540:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878803.070546:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878803.070569:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878803.070580:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878803/real 1788878803] req@ffffa06198affb80 x1875771511544832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878803.070612:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511544832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878803.070622:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878803.070625:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878803.070629:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878803.070635:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878803.070654:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878803.070680:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878803/real 1788878803] req@ffffa06198aff800 x1875771511544960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878803.070692:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511544960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878803.070699:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878803.070702:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878803.070705:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511544960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878803.709482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878803.709571:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878803.965570:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878804.733532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878804.733617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878804.989568:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878805.757535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878805.757636:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878806.013542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878806.781430:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878806.781521:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878807.037524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878807.805548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878807.805640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878808.061574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878808.189531:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878808.189546:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878808.189553:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878808.189569:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3794 00000100:00080000:0.0:1788878808.189585:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878808.189597:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878808.189653:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511545088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878808.189694:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878808.189700:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878808.189703:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878808.189708:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3794 00000100:00080000:0.0:1788878808.189713:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878808.189717:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878808.189731:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878808.189756:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511545216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878808.189780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878808.189788:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878808.189797:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878808.189799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511545088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878808.189810:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3794 00000100:00080000:0.0:1788878808.189825:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878808.189834:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878808.189846:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878808.189869:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511545344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878808.189883:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878808.189888:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878808.189892:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878808.189896:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3794 00000100:00080000:0.0:1788878808.189903:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878808.189908:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878808.189915:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878808.189931:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511545472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878808.189942:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878808.189956:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878808.189963:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878808.189966:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878808.189972:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3794 00000100:00080000:0.0:1788878808.189977:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878808.189982:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878808.189988:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878808.190001:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511545600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878808.190021:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878808/real 1788878808] req@ffffa06198afdf80 x1875771511545088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878808.190057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511545088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878808.190087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878808.190097:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878808.190105:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511545088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878808.190152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878808.190191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878808.190214:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878808/real 1788878808] req@ffffa06198afdc00 x1875771511545216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878808.190231:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511545216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878808.190242:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878808.190246:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878808.190251:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878808.190258:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878808.190278:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878808.190289:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878808/real 1788878808] req@ffffa06198afc700 x1875771511545344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878808.190303:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511545344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878808.190313:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878808.190316:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878808.190320:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878808.190329:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878808.190349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878808.190360:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878808/real 1788878808] req@ffffa06198afc000 x1875771511545472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878808.190373:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511545472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878808.190421:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878808.190426:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878808.190430:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878808.190438:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878808.190491:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878808.190513:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878808/real 1788878808] req@ffffa06198afea00 x1875771511545600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878808.190534:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511545600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878808.190545:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878808.190549:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878808.190553:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878808.829590:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878808.829684:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878809.085541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878809.853518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878809.853729:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878810.109567:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878810.877551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878810.877756:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878811.133501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878811.901529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878811.901662:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878812.157549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878812.925564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878812.925663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878813.181509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788878813.309588:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878813.309603:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788878813.309610:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878813.309626:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3800 00000100:00080000:0.0:1788878813.309654:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878813.309666:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878813.309732:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511545728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878813.309769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878813.309774:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788878813.309777:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878813.309783:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3800 00000100:00080000:0.0:1788878813.309787:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878813.309792:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878813.309806:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878813.309825:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511545856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878813.309840:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878813.309845:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788878813.309849:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878813.309859:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511545728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788878813.309861:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3800 00000100:00080000:0.0:1788878813.309873:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878813.309880:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878813.309887:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878813.309905:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511545984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878813.309918:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878813.309923:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788878813.309926:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878813.309930:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3800 00000100:00080000:0.0:1788878813.309936:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878813.309941:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878813.309946:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878813.309960:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511546112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878813.309973:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788878813.309975:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788878813.309983:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878813.309989:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878813.309996:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3800 00000100:00080000:0.0:1788878813.310002:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788878813.310008:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788878813.310015:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878813.310035:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511546240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878813.310048:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878813/real 1788878813] req@ffffa06198afe300 x1875771511545728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878813.310085:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511545728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878813.310111:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878813.310122:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878813.310133:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511545728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878813.310189:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878813.310228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878813.310247:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878813/real 1788878813] req@ffffa06198afc380 x1875771511545856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878813.310264:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511545856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878813.310276:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878813.310280:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878813.310285:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878813.310293:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878813.310314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878813.310326:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878813/real 1788878813] req@ffffa06198afca80 x1875771511545984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878813.310341:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511545984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878813.310360:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878813.310364:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878813.310369:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511545984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878813.310375:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878813.310501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878813.310528:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878813/real 1788878813] req@ffffa06198afd500 x1875771511546112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878813.310546:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511546112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878813.310557:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878813.310561:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878813.310566:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878813.310574:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878813.310602:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878813.310615:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878813/real 1788878813] req@ffffa06198afd180 x1875771511546240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878813.310629:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511546240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878813.310639:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878813.310642:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878813.310647:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878813.949533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878813.949698:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878814.205506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878814.973452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878814.973524:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878815.229478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878815.997513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878815.998700:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878816.253560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878817.021560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878817.021671:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878817.277592:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878818.045558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878818.045665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878818.301514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878818.429587:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878818.429602:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878818.429608:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878818.429624:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3805 00000100:00080000:2.0:1788878818.429640:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878818.429652:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878818.429702:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511546368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878818.429757:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878818.429761:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878818.429763:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878818.429768:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3805 00000100:00080000:2.0:1788878818.429772:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878818.429776:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878818.429787:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878818.429801:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511546496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878818.429813:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878818.429817:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878818.429818:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878818.429822:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3805 00000100:00080000:2.0:1788878818.429827:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878818.429831:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878818.429836:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878818.429841:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511546368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878818.429874:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511546624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878818.429890:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878818.429894:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878818.429897:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878818.429902:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3805 00000100:00080000:2.0:1788878818.429907:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878818.429911:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878818.429918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878818.429934:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511546752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878818.429966:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788878818.429973:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878818.429978:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878818.429984:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878818.430015:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3805 00000100:00080000:2.0:1788878818.430023:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878818.430030:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878818.430040:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878818.430055:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878818/real 1788878818] req@ffffa061888ad180 x1875771511546368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878818.430064:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511546880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878818.430107:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511546368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878818.430124:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878818.430129:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878818.430134:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511546368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878818.430144:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878818.430182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878818.430195:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878818/real 1788878818] req@ffffa061888ace00 x1875771511546496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878818.430207:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511546496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878818.430216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878818.430219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878818.430223:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878818.430248:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878818.430265:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878818.430282:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878818/real 1788878818] req@ffffa061888ac700 x1875771511546624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878818.430294:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511546624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878818.430301:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878818.430304:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878818.430307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878818.430313:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878818.430328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878818.430337:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878818/real 1788878818] req@ffffa061888aed80 x1875771511546752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878818.430348:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511546752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878818.430356:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878818.430359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878818.430362:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878818.430367:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878818.430382:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878818.430480:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878818/real 1788878818] req@ffffa061888ad880 x1875771511546880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878818.430501:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511546880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878818.430513:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878818.430517:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878818.430521:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511546880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878819.069571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878819.069650:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878819.325493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878820.093516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878820.093628:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878820.349524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878821.117556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878821.117692:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878821.373639:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878822.141527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878822.141652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878822.397473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878823.165458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878823.165565:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878823.421534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878823.549541:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878823.549556:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878823.549563:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878823.549577:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3810 00000100:00080000:3.0:1788878823.549593:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878823.549604:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878823.549698:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511547008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878823.549733:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878823.549740:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878823.549742:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878823.549749:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3810 00000100:00080000:3.0:1788878823.549754:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878823.549761:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878823.549774:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878823.549791:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511547136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878823.549806:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878823.549811:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878823.549818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878823.549822:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511547008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878823.549825:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3810 00000100:00080000:3.0:1788878823.549835:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878823.549845:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878823.549854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878823.549872:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511547264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878823.549885:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878823.549889:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878823.549892:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878823.549897:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3810 00000100:00080000:3.0:1788878823.549903:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878823.549907:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878823.549913:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878823.549931:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511547392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878823.549937:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878823.549961:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878823.549969:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878823.549973:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878823.549983:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3810 00000100:00100000:0.0:1788878823.549998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878823/real 1788878823] req@ffffa06188872a00 x1875771511547008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878823.550015:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878823.550031:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878823.550033:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511547008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878823.550048:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878823.550055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878823.550064:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878823.550071:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511547008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878823.550079:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511547520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878823.550139:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878823.550188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878823.550214:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878823/real 1788878823] req@ffffa06188873480 x1875771511547136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878823.550256:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511547136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878823.550283:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878823.550288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878823.550293:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878823.550302:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878823.550333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878823.550349:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878823/real 1788878823] req@ffffa06188871500 x1875771511547264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878823.550434:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511547264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878823.550445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878823.550450:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878823.550457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878823.550471:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878823.550514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878823.550535:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878823/real 1788878823] req@ffffa06188873100 x1875771511547392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878823.550551:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511547392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878823.550570:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878823.550574:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878823.550578:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878823.550585:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878823.550609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878823.550621:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878823/real 1788878823] req@ffffa06188872680 x1875771511547520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878823.550634:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511547520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878823.550645:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878823.550649:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878823.550653:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878824.189516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878824.189627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878824.445518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878825.213480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878825.213601:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878825.469514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878826.237545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878826.237700:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878826.493528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878827.261510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878827.261624:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878827.517573:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878828.285492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878828.285608:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878828.541482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878828.669468:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878828.669476:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878828.669481:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878828.669491:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3815 00000100:00080000:3.0:1788878828.669502:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878828.669510:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878828.669547:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511547648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878828.669568:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878828.669571:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878828.669572:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878828.669575:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3815 00000100:00080000:3.0:1788878828.669578:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878828.669582:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878828.669590:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878828.669602:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511547648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878828.669602:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511547776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878828.669616:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878828.669619:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878828.669620:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878828.669622:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3815 00000100:00080000:3.0:1788878828.669625:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878828.669628:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878828.669632:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878828.669641:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511547904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878828.669648:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878828.669650:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878828.669652:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878828.669654:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3815 00000100:00080000:3.0:1788878828.669657:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878828.669659:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878828.669662:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878828.669669:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511548032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878828.669675:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878828.669679:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878828.669681:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878828.669685:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878828.669688:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3815 00000100:00080000:3.0:1788878828.669691:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878828.669694:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878828.669697:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878828.669707:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511548160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878828.669718:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878828/real 1788878828] req@ffffa06188873800 x1875771511547648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878828.669734:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511547648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878828.669743:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878828.669746:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878828.669749:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511547648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878828.669769:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878828.669786:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878828.669795:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878828/real 1788878828] req@ffffa06188872300 x1875771511547776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878828.669803:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511547776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878828.669809:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878828.669812:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878828.669814:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878828.669818:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878828.669829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878828.669835:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878828/real 1788878828] req@ffffa06188873b80 x1875771511547904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878828.669842:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511547904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878828.669847:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878828.669849:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878828.669851:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511547904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878828.669855:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878828.669866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878828.669872:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878828/real 1788878828] req@ffffa06188870700 x1875771511548032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878828.669879:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511548032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878828.669892:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878828.669894:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878828.669897:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878828.669900:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878828.669912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878828.669918:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878828/real 1788878828] req@ffffa06188872d80 x1875771511548160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878828.669925:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511548160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878828.669931:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878828.669933:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878828.669935:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878829.309504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878829.309594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878829.565522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878830.333525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878830.333630:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878830.589489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878831.357474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878831.357537:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878831.613483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878832.381517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878832.381625:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878832.637470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878833.405527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878833.405804:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878833.661479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878833.789489:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878833.789508:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878833.789512:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878833.789524:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3820 00000100:00080000:3.0:1788878833.789554:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878833.789569:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878833.789630:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511548288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878833.789673:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878833.789678:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878833.789681:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878833.789687:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3820 00000100:00080000:3.0:1788878833.789694:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878833.789701:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878833.789716:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878833.789729:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511548288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878833.789740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511548416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878833.789756:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878833.789761:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878833.789764:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878833.789778:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3820 00000100:00080000:3.0:1788878833.789784:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878833.789789:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878833.789796:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878833.789811:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511548544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878833.789825:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878833.789828:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878833.789833:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878833.789839:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878833.789846:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3820 00000100:00080000:3.0:1788878833.789853:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878833.789859:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878833.789865:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878833.789878:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511548672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878833.789890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878833.789898:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878833/real 1788878833] req@ffffa06188871880 x1875771511548288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878833.789899:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878833.789906:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878833.789919:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3820 00000100:00080000:3.0:1788878833.789929:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878833.789934:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878833.789940:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878833.789945:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511548288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878833.789960:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511548800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878833.789963:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878833.789978:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878833.789988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511548288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878833.790023:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878833.790065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878833.790076:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878833/real 1788878833] req@ffffa06188870380 x1875771511548416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878833.790088:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511548416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878833.790096:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878833.790100:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878833.790104:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878833.790126:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878833.790150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878833.790161:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878833/real 1788878833] req@ffffa06188870a80 x1875771511548544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878833.790176:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511548544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878833.790185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878833.790188:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878833.790192:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878833.790201:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878833.790221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878833.790232:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878833/real 1788878833] req@ffffa06188871c00 x1875771511548672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878833.790248:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511548672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878833.790262:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878833.790268:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878833.790274:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878833.790310:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878833.790361:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878833.790375:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878833/real 1788878833] req@ffffa06188872d80 x1875771511548800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878833.790405:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511548800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878833.790418:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878833.790420:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878833.790423:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511548800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878834.429482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878834.429537:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878834.685534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878835.453496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878835.453606:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878835.709530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878836.477497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878836.477566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878836.733498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878837.501463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878837.501550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878837.757477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878838.525573:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878838.525701:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878838.781548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878838.909644:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878838.909658:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878838.909665:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878838.909679:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3825 00000100:00080000:1.0:1788878838.909696:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878838.909708:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878838.909765:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511548928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878838.909803:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878838.909807:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878838.909809:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878838.909814:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3825 00000100:00080000:1.0:1788878838.909818:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878838.909824:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878838.909836:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878838.909864:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511549056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878838.909879:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878838.909884:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878838.909886:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878838.909890:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3825 00000100:00080000:1.0:1788878838.909894:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878838.909907:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878838.909912:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878838.909914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511548928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878838.909928:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511549184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878838.909942:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878838.909957:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878838.909960:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878838.909964:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3825 00000100:00080000:1.0:1788878838.909967:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878838.909971:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878838.909976:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878838.909991:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511549312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878838.910002:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878838.910006:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878838.910008:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878838.910013:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3825 00000100:00080000:1.0:1788878838.910038:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878838.910044:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878838.910050:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878838.910074:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511549440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878838.910143:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878838.910206:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878838.910286:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878838.910309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878839.549515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878839.549629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878839.805588:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878840.573541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878840.573699:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878840.829516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878841.597502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878841.597613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878841.853486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:0.0:1788878841.981721:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:0.0:1788878841.981769:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788878841.981788:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:0.0:1788878841.981837:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:0.0:1788878841.981847:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:0.0:1788878841.981854:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:0.0:1788878841.981863:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788878841.981895:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878838/real 1788878841] req@ffffa0618869bb80 x1875771511549056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878841.981927:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511549056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878841.981949:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878841.981955:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878841.981960:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878841.981970:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878838/real 1788878841] req@ffffa0618869ad80 x1875771511549184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878841.981984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511549184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878841.981993:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878841.981997:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878841.982001:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878841.982009:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878838/real 1788878841] req@ffffa06188698000 x1875771511549312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878841.982021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511549312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878841.982035:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878841.982040:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878841.982043:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878841.982050:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878838/real 1788878841] req@ffffa06188699f80 x1875771511549440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878841.982061:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511549440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878841.982069:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878841.982072:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878841.982075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878841.982082:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878838/real 1788878841] req@ffffa0618869a680 x1875771511548928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878841.982095:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511548928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878841.982104:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878841.982108:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878841.982111:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511548928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878842.621516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878842.621555:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878842.877541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878843.645503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878843.645675:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878843.901518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878844.029515:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878844.029525:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878844.029529:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878844.029541:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3830 00000100:00080000:1.0:1788878844.029552:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878844.029561:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878844.029598:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511549568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878844.029622:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878844.029625:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878844.029627:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878844.029631:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3830 00000100:00080000:1.0:1788878844.029634:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878844.029638:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878844.029649:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878844.029683:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511549568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878844.029686:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511549696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878844.029718:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878844.029726:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878844.029730:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878844.029738:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3830 00000100:00080000:1.0:1788878844.029747:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878844.029787:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878844.029795:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788878844.029814:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878844.029815:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511549824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878844.029831:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878844.029887:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788878844.029893:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878844/real 1788878844] req@ffffa0618869a680 x1875771511549568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878844.029894:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878844.029911:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3830 00000100:00080000:1.0:1788878844.029927:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878844.029941:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878844.029953:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878844.029957:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511549568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878844.029984:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788878844.029986:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511549952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878844.030002:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878844.030022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511549568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878844.030032:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878844.030042:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878844.030047:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878844.030058:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3830 00000100:00080000:1.0:1788878844.030067:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878844.030079:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549696:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788878844.030080:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878844.030098:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878844.030131:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511550080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878844.030160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878844.030198:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878844/real 1788878844] req@ffffa06188699f80 x1875771511549696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878844.030255:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511549696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878844.030272:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878844.030278:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878844.030285:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878844.030298:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878844.030344:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878844.030369:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878844/real 1788878844] req@ffffa06188698000 x1875771511549824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878844.030438:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511549824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878844.030456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878844.030462:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878844.030469:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878844.030482:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878844.030525:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878844.030549:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878844/real 1788878844] req@ffffa0618869ad80 x1875771511549952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878844.030590:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511549952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878844.030608:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878844.030614:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878844.030635:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511549952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878844.030675:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878844.030717:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878844.031701:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878844/real 1788878844] req@ffffa0618869bb80 x1875771511550080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878844.031766:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511550080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878844.031785:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878844.031792:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878844.031798:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878844.669549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878844.669595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878844.925539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878845.693446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878845.693622:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878845.949574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878846.717479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878846.717613:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878846.973564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878847.741522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878847.741667:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878847.997501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878848.765544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878848.765724:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878849.021509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878849.149523:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878849.149539:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878849.149547:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878849.149583:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3835 00000100:00080000:2.0:1788878849.149606:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878849.149620:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878849.149696:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511550208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878849.149754:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878849.149761:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878849.149763:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878849.149771:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3835 00000100:00080000:2.0:1788878849.149777:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878849.149783:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878849.149798:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878849.149827:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511550336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878849.149857:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878849.149861:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878849.149864:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878849.149868:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3835 00000100:00080000:2.0:1788878849.149872:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878849.149879:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878849.149886:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878849.149901:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511550464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878849.149914:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878849.149919:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788878849.149920:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511550208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878849.149922:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878849.149938:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3835 00000100:00080000:2.0:1788878849.149948:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878849.149952:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878849.149957:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878849.149974:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511550592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878849.149984:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878849.149988:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878849.149990:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878849.149995:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3835 00000100:00080000:2.0:1788878849.150011:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878849.150016:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878849.150022:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878849.150029:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878849.150040:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511550720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878849.150106:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878849/real 1788878849] req@ffffa061888ac380 x1875771511550208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878849.150137:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511550208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878849.150172:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878849.150194:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878849.150200:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511550208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878849.150211:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878849.150243:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878849.150257:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878849/real 1788878849] req@ffffa061888ae680 x1875771511550336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878849.150268:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511550336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878849.150276:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878849.150279:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878849.150282:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878849.150308:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878849.150323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878849.150332:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878849/real 1788878849] req@ffffa061888af100 x1875771511550464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878849.150341:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511550464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878849.150348:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878849.150350:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878849.150353:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878849.150359:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878849.150372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878849.150411:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878849/real 1788878849] req@ffffa061888ae300 x1875771511550592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878849.150422:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511550592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878849.150429:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878849.150432:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878849.150435:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878849.150440:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878849.150456:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878849.150465:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878849/real 1788878849] req@ffffa061888aea00 x1875771511550720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878849.150474:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511550720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878849.150481:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878849.150484:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878849.150486:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878849.789483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878849.789568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878850.045480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878850.813522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878850.813600:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878851.069529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878851.837483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878851.837556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878852.093588:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878852.861478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878852.861564:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878853.117519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878853.885570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878853.885707:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878854.141584:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878854.269581:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878854.269606:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878854.269612:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878854.269626:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3840 00000100:00080000:3.0:1788878854.269640:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878854.269651:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878854.269702:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511550848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878854.269755:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878854.269762:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878854.269768:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878854.269779:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3840 00000100:00080000:3.0:1788878854.269787:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878854.269810:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878854.269826:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511550848:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878854.269833:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878854.269869:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511550976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878854.269890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878854.269895:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878854.269898:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878854.269905:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3840 00000100:00080000:3.0:1788878854.269912:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878854.269919:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878854.269926:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878854.269947:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511551104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878854.269961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878854.270007:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878854.270016:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878854.270022:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878854.270035:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3840 00000100:00080000:3.0:1788878854.270044:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878854.270062:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878854/real 1788878854] req@ffffa06188870700 x1875771511550848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878854.270071:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878854.270098:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878854.270101:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511550848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878854.270124:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511551232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878854.270125:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878854.270162:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878854.270180:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511550848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878854.270197:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878854.270203:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878854.270207:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878854.270215:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3840 00000100:00080000:3.0:1788878854.270224:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878854.270238:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878854.270239:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550976:192.168.203.111@tcp:38:kworker.0 00000008:00100000:3.0:1788878854.270257:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878854.270282:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511551360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878854.270285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878854.270309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878854/real 1788878854] req@ffffa06188873b80 x1875771511550976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878854.270321:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511550976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878854.270330:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878854.270335:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878854.270339:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511550976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878854.270347:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878854.270362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878854.270369:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878854/real 1788878854] req@ffffa06188872300 x1875771511551104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878854.270377:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511551104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878854.270405:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878854.270407:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878854.270409:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878854.270415:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878854.270430:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878854.270438:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878854/real 1788878854] req@ffffa06188873800 x1875771511551232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878854.270449:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511551232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878854.270465:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878854.270469:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878854.270472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878854.270493:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878854.270512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878854.270521:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878854/real 1788878854] req@ffffa06188872680 x1875771511551360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878854.270529:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511551360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878854.270535:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878854.270537:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878854.270539:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878854.909510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878854.909655:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878855.165557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878855.933510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878855.933589:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878856.189452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878856.957512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878856.957664:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878857.213471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878857.981538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878857.981573:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878858.237519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878859.005523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878859.005695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878859.261480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878859.389589:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878859.389604:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878859.389612:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878859.389628:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3846 00000100:00080000:1.0:1788878859.389645:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878859.389658:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878859.389737:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511551488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878859.389788:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878859.389795:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878859.389800:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878859.389809:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3846 00000100:00080000:1.0:1788878859.389817:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878859.389826:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878859.389843:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878859.389850:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511551488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878859.389866:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511551616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878859.389884:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878859.389890:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878859.389893:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878859.389899:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3846 00000100:00080000:1.0:1788878859.389932:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878859.389940:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878859.389947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788878859.389950:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878859.389968:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511551744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878859.389985:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878859.389991:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878859.389994:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878859.390001:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3846 00000100:00100000:0.0:1788878859.390002:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878859/real 1788878859] req@ffffa0618869aa00 x1875771511551488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878859.390007:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878859.390014:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878859.390023:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878859.390034:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511551488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878859.390040:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511551872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878859.390058:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878859.390063:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878859.390066:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878859.390067:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878859.390072:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3846 00000100:00080000:0.0:1788878859.390074:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788878859.390077:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878859.390082:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511551488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878859.390084:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878859.390092:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878859.390129:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511552000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878859.390143:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878859.390202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878859.390227:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878859/real 1788878859] req@ffffa0618869b800 x1875771511551616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878859.390242:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511551616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878859.390253:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878859.390257:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878859.390261:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878859.390268:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878859.390293:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878859.390304:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878859/real 1788878859] req@ffffa06188699c00 x1875771511551744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878859.390317:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511551744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878859.390327:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878859.390330:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878859.390333:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878859.390341:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878859.390359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878859.390370:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878859/real 1788878859] req@ffffa06188698380 x1875771511551872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878859.390416:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511551872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878859.390427:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878859.390430:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878859.390434:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511551872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878859.390456:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878859.390478:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878859.390490:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878859/real 1788878859] req@ffffa0618869b100 x1875771511552000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878859.390503:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511552000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878859.390512:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878859.390516:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878859.390520:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878860.029506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878860.029620:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878860.285513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878861.053570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878861.053634:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878861.309495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878862.077482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878862.077640:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878862.333551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878863.101486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878863.101667:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878863.357529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878864.125459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878864.125493:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878864.381453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878864.509474:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878864.509485:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878864.509489:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878864.509501:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3851 00000100:00080000:1.0:1788878864.509511:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878864.509519:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878864.509559:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511552128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878864.509592:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878864.509596:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878864.509599:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878864.509604:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3851 00000100:00080000:1.0:1788878864.509608:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878864.509613:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878864.509623:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878864.509626:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511552128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878864.509635:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511552256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878864.509645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878864.509649:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878864.509650:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878864.509654:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3851 00000100:00080000:1.0:1788878864.509657:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878864.509661:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878864.509666:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878864.509675:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511552384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878864.509689:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878864.509692:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788878864.509694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878864.509694:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878864.509698:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3851 00000100:00080000:1.0:1788878864.509700:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878864.509704:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878864.509709:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878864.509719:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511552512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878864.509727:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878864.509728:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878864/real 1788878864] req@ffffa0618869b100 x1875771511552128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878864.509731:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878864.509733:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878864.509737:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3851 00000100:00080000:1.0:1788878864.509740:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878864.509744:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878864.509747:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511552128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878864.509749:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878864.509758:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511552640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878864.509759:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878864.509763:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878864.509767:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511552128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878864.509805:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878864.509825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878864.509835:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878864/real 1788878864] req@ffffa06188698380 x1875771511552256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878864.509843:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511552256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878864.509849:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878864.509851:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878864.509853:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878864.509857:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878864.509868:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878864.509874:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878864/real 1788878864] req@ffffa06188699c00 x1875771511552384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878864.509886:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511552384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878864.509891:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878864.509893:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878864.509894:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878864.509898:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878864.509908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878864.509922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878864/real 1788878864] req@ffffa0618869b800 x1875771511552512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878864.509930:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511552512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878864.509936:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878864.509938:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878864.509940:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878864.509943:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878864.509963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878864.509969:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878864/real 1788878864] req@ffffa0618869aa00 x1875771511552640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878864.509980:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511552640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878864.509990:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878864.509992:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878864.509994:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878865.149505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878865.149600:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878865.405510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878866.173473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878866.173523:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878866.429460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878867.197497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878867.197541:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878867.453487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878868.221504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878868.221581:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878868.477476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878869.245503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878869.245567:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878869.501456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878869.629492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878869.629502:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878869.629507:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878869.629518:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3856 00000100:00080000:3.0:1788878869.629530:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878869.629538:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878869.629578:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511552768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878869.629603:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878869.629607:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878869.629609:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878869.629613:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3856 00000100:00080000:3.0:1788878869.629616:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878869.629620:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878869.629631:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878869.629644:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511552768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878869.629645:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511552896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878869.629661:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878869.629664:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878869.629666:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878869.629669:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3856 00000100:00080000:3.0:1788878869.629672:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878869.629675:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878869.629679:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878869.629689:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511553024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878869.629698:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878869.629701:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878869.629710:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878869.629714:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3856 00000100:00080000:3.0:1788878869.629717:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878869.629722:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878869.629725:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788878869.629728:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878869.629740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511553152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878869.629748:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878869.629751:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878869.629753:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878869.629756:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3856 00000100:00080000:3.0:1788878869.629759:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878869.629761:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878869.629766:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878869.629782:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878869/real 1788878869] req@ffffa06188873100 x1875771511552768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878869.629782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511553280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878869.629824:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511552768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878869.629835:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878869.629839:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878869.629842:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511552768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878869.629864:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878869.629883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878869.629893:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878869/real 1788878869] req@ffffa06188871500 x1875771511552896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878869.629903:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511552896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878869.629915:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878869.629918:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878869.629921:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511552896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878869.629925:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878869.629943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878869.629957:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878869/real 1788878869] req@ffffa06188873480 x1875771511553024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878869.629972:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511553024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878869.629982:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878869.629985:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878869.629988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878869.629993:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878869.630020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878869.630029:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878869/real 1788878869] req@ffffa06188872a00 x1875771511553152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878869.630055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511553152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878869.630078:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878869.630081:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878869.630084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878869.630089:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878869.630105:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878869.630114:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878869/real 1788878869] req@ffffa06188872680 x1875771511553280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878869.630122:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511553280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878869.630129:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878869.630131:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878869.630133:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878870.269572:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878870.269624:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878870.525574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878871.293463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878871.293541:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878871.549457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878872.317530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878872.317626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878872.573516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878873.341597:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878873.341609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878873.597530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:0.0:1788878874.365567:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788878874.365575:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788878874.621518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878874.749544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878874.749557:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878874.749562:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878874.749574:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3861 00000100:00080000:1.0:1788878874.749587:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878874.749594:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878874.749640:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511553408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878874.749681:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878874.749684:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878874.749685:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878874.749689:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3861 00000100:00080000:1.0:1788878874.749692:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878874.749695:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878874.749705:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878874.749720:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511553536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878874.749724:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511553408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878874.749743:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878874.749748:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878874.749750:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878874.749755:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3861 00000100:00080000:1.0:1788878874.749760:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878874.749765:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878874.749771:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878874.749782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511553664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878874.749790:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878874.749793:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788878874.749798:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878874.749798:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878874.749805:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3861 00000100:00080000:1.0:1788878874.749809:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878874.749812:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878874.749816:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878874.749825:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511553792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878874.749832:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878874.749835:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878874.749837:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878874.749842:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878874/real 1788878874] req@ffffa0618869bb80 x1875771511553408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878874.749842:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3861 00000100:00080000:1.0:1788878874.749844:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878874.749852:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878874.749860:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878874.749886:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511553408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878874.749887:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511553920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878874.749913:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878874.749918:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878874.749921:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511553408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878874.749964:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878874.749989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878874.750016:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878874/real 1788878874] req@ffffa0618869ad80 x1875771511553536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878874.750048:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511553536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878874.750058:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878874.750062:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878874.750064:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878874.750069:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878874.750103:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878874.750117:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878874/real 1788878874] req@ffffa06188698000 x1875771511553664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878874.750129:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511553664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878874.750137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878874.750141:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878874.750144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878874.750149:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878874.750179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878874.750186:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878874/real 1788878874] req@ffffa06188699f80 x1875771511553792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878874.750194:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511553792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878874.750200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878874.750202:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878874.750204:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878874.750208:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878874.750218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878874.750224:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878874/real 1788878874] req@ffffa0618869a680 x1875771511553920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878874.750232:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511553920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878874.750237:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878874.750239:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878874.750241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511553920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878875.389515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878875.389596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878875.645535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878876.413552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878876.413643:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878876.669533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878877.437522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878877.437647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878877.693536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878878.461494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878878.461566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878878.717470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:0.0:1788878879.485586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788878879.485644:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788878879.741492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878879.869512:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878879.869523:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878879.869527:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878879.869540:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3866 00000100:00080000:2.0:1788878879.869558:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878879.869568:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878879.869611:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511554048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878879.869645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878879.869649:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878879.869650:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878879.869654:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3866 00000100:00080000:2.0:1788878879.869657:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878879.869662:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878879.869674:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878879.869692:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511554176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878879.869701:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878879.869706:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878879.869708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878879.869712:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3866 00000100:00080000:2.0:1788878879.869714:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878879.869717:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878879.869721:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878879.869730:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511554304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878879.869738:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878879.869741:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878879.869743:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878879.869749:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3866 00000100:00080000:2.0:1788878879.869752:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878879.869757:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878879.869762:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878879.869775:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511554432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878879.869784:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878879.869786:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878879.869788:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878879.869793:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3866 00000100:00080000:2.0:1788878879.869800:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878879.869804:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878879.869804:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511554048:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878879.869822:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878879.869834:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511554560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878879.869921:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878879.869997:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878879/real 1788878879] req@ffffa061888afb80 x1875771511554048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878879.870026:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511554048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878879.870051:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878879.870057:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878879.870063:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511554048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878879.870073:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878879.870103:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878879.870119:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878879/real 1788878879] req@ffffa061888adc00 x1875771511554176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878879.870134:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511554176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878879.870145:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878879.870148:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878879.870152:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878879.870160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878879.870181:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878879.870193:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878879/real 1788878879] req@ffffa061888ad500 x1875771511554304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878879.870210:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511554304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878879.870221:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878879.870226:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878879.870233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878879.870289:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878879.870311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878879.870324:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878879/real 1788878879] req@ffffa061888ac000 x1875771511554432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878879.870338:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511554432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878879.870359:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878879.870362:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878879.870366:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878879.870373:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878879.870469:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878879.870486:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878879/real 1788878879] req@ffffa061888adf80 x1875771511554560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878879.870501:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511554560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878879.870511:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878879.870515:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878879.870519:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878880.509545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878880.509576:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878880.765481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878881.533589:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878881.533609:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878881.789510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878882.557466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878882.557512:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878882.813455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878883.581486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878883.581557:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878883.837509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878884.605564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878884.605668:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878884.861491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878884.989445:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878884.989454:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878884.989458:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878884.989469:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3871 00000100:00080000:2.0:1788878884.989482:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878884.989490:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878884.989535:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511554688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878884.989558:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878884.989561:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878884.989562:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878884.989565:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3871 00000100:00080000:2.0:1788878884.989568:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878884.989571:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878884.989603:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878884.989620:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511554816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878884.989636:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878884.989641:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878884.989660:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878884.989668:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3871 00000100:00080000:2.0:1788878884.989674:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878884.989677:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878884.989685:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878884.989699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511554944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878884.989714:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511554688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878884.989719:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878884.989740:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878884.989743:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878884.989748:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3871 00000100:00080000:2.0:1788878884.989754:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878884.989761:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878884.989768:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878884.989788:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511555072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878884.989821:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878884.989828:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878884.989833:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788878884.989844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878884.989850:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3871 00000100:00080000:2.0:1788878884.989858:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878884.989864:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878884.989874:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878884.989912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878884/real 1788878884] req@ffffa061888aca80 x1875771511554688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878884.989925:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511555200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878884.989971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511554688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878884.989997:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878884.990004:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878884.990009:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511554688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878884.990019:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878884.990045:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878884.990057:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878884/real 1788878884] req@ffffa061888af480 x1875771511554816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878884.990069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511554816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878884.990077:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878884.990080:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878884.990083:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878884.990115:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878884.990132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878884.990141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878884/real 1788878884] req@ffffa061888adf80 x1875771511554944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878884.990153:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511554944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878884.990167:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878884.990171:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878884.990174:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511554944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878884.990180:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878884.990195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878884.990203:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878884/real 1788878884] req@ffffa061888ac000 x1875771511555072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878884.990214:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511555072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878884.990222:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878884.990224:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878884.990227:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878884.990232:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878884.990247:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878884.990256:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878884/real 1788878884] req@ffffa061888ad500 x1875771511555200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878884.990266:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511555200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878884.990273:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878884.990276:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878884.990280:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878885.629554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878885.629612:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788878885.885472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878886.653493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878886.653547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878886.909514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878887.677437:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878887.677551:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878887.933505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878888.701556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878888.701664:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878888.957486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878889.725515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878889.725674:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878889.981502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878890.109576:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878890.109591:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878890.109599:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878890.109617:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3876 00000100:00080000:1.0:1788878890.109645:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878890.109657:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878890.109719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511555328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878890.109761:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878890.109767:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878890.109770:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878890.109775:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3876 00000100:00080000:1.0:1788878890.109779:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878890.109784:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878890.109797:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878890.109814:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511555456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878890.109827:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878890.109832:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878890.109835:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878890.109839:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3876 00000100:00080000:1.0:1788878890.109842:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878890.109847:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878890.109853:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878890.109877:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511555584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878890.109918:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878890.109921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511555328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878890.109952:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878890.109956:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878890.109963:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3876 00000100:00080000:1.0:1788878890.109968:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878890.109975:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878890.109984:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878890.110002:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511555712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878890.110015:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878890.110019:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878890.110021:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878890.110025:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3876 00000100:00080000:1.0:1788878890.110033:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878890.110038:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878890.110044:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878890.110059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878890.110061:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511555840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878890.110132:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878890/real 1788878890] req@ffffa0618869a680 x1875771511555328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878890.110156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511555328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878890.110169:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878890.110174:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878890.110179:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511555328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878890.110188:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878890.110225:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878890.110238:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878890/real 1788878890] req@ffffa06188699f80 x1875771511555456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878890.110250:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511555456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878890.110259:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878890.110262:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878890.110265:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878890.110293:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878890.110310:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878890.110321:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878890/real 1788878890] req@ffffa06188698000 x1875771511555584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878890.110349:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511555584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878890.110360:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878890.110364:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878890.110368:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878890.110376:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878890.110459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878890.110477:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878890/real 1788878890] req@ffffa0618869ad80 x1875771511555712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878890.110509:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511555712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878890.110518:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878890.110521:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878890.110524:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878890.110530:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878890.110553:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878890.110575:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878890/real 1788878890] req@ffffa0618869bb80 x1875771511555840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878890.110588:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511555840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878890.110598:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878890.110601:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878890.110604:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511555840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878890.749468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878890.749590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878891.005482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878891.773507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878891.773605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878892.029516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878892.797553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878892.797693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878893.053544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878893.821514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878893.821604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878894.077538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:0.0:1788878894.845597:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788878894.845645:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788878895.101509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878895.229511:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878895.229529:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878895.229536:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878895.229554:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3881 00000100:00080000:1.0:1788878895.229573:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878895.229585:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878895.229645:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511555968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878895.229687:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878895.229692:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878895.229696:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878895.229702:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3881 00000100:00080000:1.0:1788878895.229707:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878895.229712:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878895.229737:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878895.229757:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511556096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878895.229771:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878895.229776:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878895.229779:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878895.229784:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3881 00000100:00080000:1.0:1788878895.229788:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878895.229792:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878895.229800:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878895.229815:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511556224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878895.229833:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878895.229837:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878895.229844:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878895.229846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511555968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878895.229855:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3881 00000100:00080000:1.0:1788878895.229895:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878895.229901:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878895.229910:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878895.229932:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511556352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878895.229945:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878895.229951:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878895.229953:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878895.229958:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3881 00000100:00080000:1.0:1788878895.229963:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878895.229968:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878895.229976:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878895.229993:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878895.229995:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511556480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878895.230067:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878895/real 1788878895] req@ffffa0618869aa00 x1875771511555968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878895.230094:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511555968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878895.230110:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878895.230115:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878895.230121:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511555968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878895.230131:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878895.230160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878895.230174:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878895/real 1788878895] req@ffffa0618869b800 x1875771511556096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878895.230190:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511556096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878895.230200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878895.230204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878895.230208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878895.230242:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878895.230263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878895.230275:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878895/real 1788878895] req@ffffa06188699c00 x1875771511556224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878895.230289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511556224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878895.230299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878895.230302:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878895.230306:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878895.230313:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878895.230332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878895.230343:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878895/real 1788878895] req@ffffa06188698380 x1875771511556352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878895.230357:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511556352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878895.230367:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878895.230371:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878895.230374:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878895.230430:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878895.230456:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878895.230471:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878895/real 1788878895] req@ffffa0618869b100 x1875771511556480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878895.230501:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511556480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878895.230512:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878895.230516:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878895.230520:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878895.869523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878895.869657:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878896.125566:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878896.893526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878896.893649:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878897.149552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878897.917557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878897.917661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878898.173484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878898.941502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878898.941604:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878899.197524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878899.965549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878899.965611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878900.221462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878900.349498:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878900.349513:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878900.349520:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878900.349535:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3887 00000100:00080000:1.0:1788878900.349553:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878900.349561:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878900.349637:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511556608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878900.349666:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878900.349671:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878900.349673:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878900.349680:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3887 00000100:00080000:1.0:1788878900.349686:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878900.349693:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878900.349707:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878900.349726:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511556608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878900.349734:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511556736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878900.349753:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878900.349757:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878900.349761:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878900.349766:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3887 00000100:00080000:1.0:1788878900.349773:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878900.349780:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878900.349788:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878900.349806:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511556864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878900.349816:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878900.349844:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878900.349856:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878900.349859:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878900.349876:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3887 00000100:00080000:1.0:1788878900.349881:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878900.349886:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878900.349894:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878900.349898:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878900/real 1788878900] req@ffffa0618869b100 x1875771511556608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878900.349917:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511556992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878900.349923:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511556608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878900.349964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878900.349966:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878900.349971:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788878900.349980:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788878900.349981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511556608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878900.349988:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878900.349998:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3887 00000100:00080000:1.0:1788878900.350005:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878900.350011:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878900.350016:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556736:192.168.203.111@tcp:38:kworker.0 00000008:00100000:1.0:1788878900.350021:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878900.350048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878900.350049:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511557120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878900.350062:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878900/real 1788878900] req@ffffa06188698380 x1875771511556736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878900.350080:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511556736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878900.350096:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878900.350100:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878900.350104:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878900.350110:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878900.350125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878900.350133:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878900/real 1788878900] req@ffffa06188699c00 x1875771511556864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878900.350143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511556864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878900.350150:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878900.350153:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878900.350155:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878900.350160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878900.350175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878900.350182:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878900/real 1788878900] req@ffffa0618869b800 x1875771511556992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878900.350192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511556992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878900.350199:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878900.350201:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878900.350205:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511556992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878900.350219:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878900.350233:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878900.350241:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878900/real 1788878900] req@ffffa0618869aa00 x1875771511557120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878900.350251:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511557120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878900.350258:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878900.350260:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878900.350263:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878900.989535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878900.989683:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878901.245547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878902.013515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878902.013670:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878902.269516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878903.037486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878903.037549:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878903.293500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878904.061594:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878904.061773:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878904.317465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878905.085591:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878905.085756:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878905.341493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878905.469533:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878905.469542:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878905.469546:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878905.469557:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3892 00000100:00080000:3.0:1788878905.469568:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878905.469575:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878905.469611:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511557248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878905.469638:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878905.469641:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878905.469642:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878905.469645:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3892 00000100:00080000:3.0:1788878905.469647:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878905.469651:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878905.469660:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878905.469670:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511557376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878905.469679:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878905.469681:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878905.469683:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878905.469686:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3892 00000100:00080000:3.0:1788878905.469688:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878905.469697:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878905.469700:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511557248:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878905.469703:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878905.469720:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511557504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878905.469727:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878905.469729:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878905.469731:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878905.469735:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3892 00000100:00080000:3.0:1788878905.469737:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878905.469740:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878905.469744:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878905.469753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511557632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878905.469758:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878905.469760:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878905.469761:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878905.469764:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3892 00000100:00080000:3.0:1788878905.469766:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878905.469769:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878905.469773:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878905.469781:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511557760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878905.469883:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878905.469927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878905.469955:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878905.469981:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878906.109549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878906.109674:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878906.365532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878907.133506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878907.133677:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878907.389465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878908.157520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878908.157652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878908.413535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878908.541700:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878908.541746:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878908.541763:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878908.541826:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878908.541834:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878908.541842:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878908.541852:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878908.541920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878905/real 1788878908] req@ffffa06188872300 x1875771511557376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878908.541982:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511557376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878908.542009:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878908.542019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878908.542028:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878908.542045:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878905/real 1788878908] req@ffffa06188873b80 x1875771511557504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878908.542070:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511557504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878908.542084:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878908.542092:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878908.542101:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878908.542114:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878905/real 1788878908] req@ffffa06188870700 x1875771511557632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878908.542142:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511557632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878908.542158:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878908.542165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878908.542172:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878908.542184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878905/real 1788878908] req@ffffa06188872d80 x1875771511557760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878908.542207:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511557760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878908.542224:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878908.542229:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878908.542237:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511557760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878908.542251:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878905/real 1788878908] req@ffffa06188873800 x1875771511557248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878908.542275:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511557248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878908.542289:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878908.542295:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878908.542302:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511557248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878909.181596:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878909.181704:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878909.437470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878910.205529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878910.205599:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878910.461535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878910.589619:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878910.589632:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878910.589638:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878910.589652:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3897 00000100:00080000:2.0:1788878910.589666:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878910.589676:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878910.589726:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511557888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878910.589804:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878910.589808:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878910.589811:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878910.589816:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3897 00000100:00080000:2.0:1788878910.589823:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878910.589829:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878910.589841:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878910.589874:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511557888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878910.589877:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511558016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878910.589910:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878910.589914:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878910.589915:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878910.589921:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3897 00000100:00080000:2.0:1788878910.589924:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878910.589929:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878910.589935:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878910.589967:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511558144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878910.589976:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878910.589980:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878910.589982:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878910.589988:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3897 00000100:00080000:2.0:1788878910.589993:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878910.589996:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878910.590001:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878910.590003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878910.590035:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511558272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878910.590061:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878910/real 1788878910] req@ffffa061888adc00 x1875771511557888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878910.590105:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511557888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878910.590107:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878910.590138:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788878910.590144:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878910.590145:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788878910.590157:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878910.590168:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511557888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878910.590171:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3897 00000100:00080000:2.0:1788878910.590186:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878910.590196:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878910.590209:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878910.590219:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878910.590242:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511558400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878910.590274:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878910.590304:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878910/real 1788878910] req@ffffa061888afb80 x1875771511558016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878910.590340:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511558016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878910.590355:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878910.590361:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878910.590367:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878910.590379:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878910.590456:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878910.590474:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878910/real 1788878910] req@ffffa061888aea00 x1875771511558144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878910.590490:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511558144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878910.590501:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878910.590506:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878910.590511:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878910.590521:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878910.590559:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878910.590576:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878910/real 1788878910] req@ffffa061888ae300 x1875771511558272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878910.590593:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511558272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878910.590605:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878910.590610:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878910.590615:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878910.590643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878910.590674:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878910.590691:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878910/real 1788878910] req@ffffa061888af100 x1875771511558400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878910.590708:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511558400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878910.590720:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878910.590725:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878910.590730:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878911.229511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878911.229629:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878911.485460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878912.253536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878912.253654:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878912.509508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878913.277467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878913.277550:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878913.533486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878914.301554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878914.301683:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878914.557460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878915.325458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878915.325542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878915.581469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878915.709487:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878915.709498:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878915.709502:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878915.709512:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3902 00000100:00080000:2.0:1788878915.709523:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878915.709532:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878915.709571:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511558528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878915.709594:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878915.709598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878915.709601:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878915.709606:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3902 00000100:00080000:2.0:1788878915.709610:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878915.709616:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878915.709626:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878915.709628:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511558528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878915.709639:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511558656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878915.709649:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878915.709652:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878915.709654:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878915.709657:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3902 00000100:00080000:2.0:1788878915.709661:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878915.709665:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878915.709669:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878915.709679:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511558784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878915.709688:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878915.709691:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878915.709693:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878915.709694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878915.709697:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3902 00000100:00080000:2.0:1788878915.709700:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878915.709703:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878915.709708:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878915.709731:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511558912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878915.709741:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878915/real 1788878915] req@ffffa061888ae680 x1875771511558528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878915.709741:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878915.709744:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878915.709746:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878915.709750:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3902 00000100:00080000:2.0:1788878915.709754:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878915.709757:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878915.709760:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511558528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878915.709762:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878915.709771:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788878915.709773:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511559040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878915.709776:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878915.709780:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511558528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878915.709804:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878915.709827:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878915.709837:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878915/real 1788878915] req@ffffa061888ac380 x1875771511558656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878915.709845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511558656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878915.709851:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878915.709854:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878915.709856:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878915.709860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878915.709871:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878915.709877:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878915/real 1788878915] req@ffffa061888ad880 x1875771511558784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878915.709884:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511558784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878915.709889:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878915.709891:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878915.709893:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878915.709896:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878915.709906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878915.709912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878915/real 1788878915] req@ffffa061888aed80 x1875771511558912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878915.709920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511558912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878915.709926:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878915.709927:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878915.709929:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511558912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878915.709932:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878915.709942:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878915.709947:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878915/real 1788878915] req@ffffa061888ac700 x1875771511559040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878915.709955:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511559040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878915.709960:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878915.709962:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878915.709963:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878916.349469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878916.349564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878916.605495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878917.373449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878917.373511:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878917.629472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878918.397482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878918.397532:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878918.653450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878919.421459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878919.421640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878919.677522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878920.445470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878920.445576:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878920.701470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878920.829505:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878920.829515:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878920.829520:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878920.829530:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3907 00000100:00080000:1.0:1788878920.829543:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878920.829551:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878920.829588:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511559168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878920.829613:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878920.829616:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878920.829617:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878920.829621:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3907 00000100:00080000:1.0:1788878920.829625:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878920.829628:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878920.829637:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878920.829650:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511559296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878920.829658:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511559168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878920.829663:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878920.829671:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878920.829673:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878920.829676:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3907 00000100:00080000:1.0:1788878920.829678:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878920.829680:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878920.829702:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878920.829714:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511559424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878920.829721:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878920.829723:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878920.829724:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878920.829727:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3907 00000100:00080000:1.0:1788878920.829750:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878920.829754:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878920.829759:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878920.829766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878920.829775:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511559552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878920.829784:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878920.829787:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878920.829790:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878920.829794:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3907 00000100:00080000:1.0:1788878920.829797:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878920.829799:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878920.829802:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878920.829813:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511559680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878920.829822:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878920/real 1788878920] req@ffffa0618869bb80 x1875771511559168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878920.829850:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511559168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878920.829860:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878920.829864:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878920.829867:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511559168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878920.829892:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878920.829913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878920.829922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878920/real 1788878920] req@ffffa0618869ad80 x1875771511559296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878920.829931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511559296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878920.829937:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878920.829940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878920.829942:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878920.829946:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878920.829960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878920.829970:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878920/real 1788878920] req@ffffa06188698000 x1875771511559424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878920.829984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511559424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878920.829993:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878920.829996:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878920.830000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878920.830006:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878920.830025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878920.830034:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878920/real 1788878920] req@ffffa06188699f80 x1875771511559552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878920.830042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511559552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878920.830048:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878920.830050:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878920.830052:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878920.830056:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878920.830066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878920.830072:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878920/real 1788878920] req@ffffa0618869a680 x1875771511559680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878920.830079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511559680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878920.830094:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878920.830098:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878920.830102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878921.469488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878921.469624:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878921.725520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878922.493475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878922.493529:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878922.749452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878923.517477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878923.517539:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878923.773462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878924.541464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878924.541544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878924.797459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878925.565458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878925.565516:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878925.821466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878925.949478:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878925.949486:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878925.949490:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878925.949500:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3912 00000100:00080000:1.0:1788878925.949511:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878925.949518:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878925.949554:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511559808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878925.949581:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878925.949586:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878925.949588:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878925.949592:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3912 00000100:00080000:1.0:1788878925.949595:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878925.949598:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878925.949608:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878925.949617:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511559936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878925.949627:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511559808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788878925.949627:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878925.949633:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878925.949637:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878925.949641:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3912 00000100:00080000:1.0:1788878925.949643:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878925.949645:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878925.949662:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878925.949672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511560064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878925.949679:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878925.949682:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878925.949684:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878925.949686:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3912 00000100:00080000:1.0:1788878925.949705:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878925.949708:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878925.949713:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878925.949719:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788878925.949725:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511560192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878925.949734:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878925.949736:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878925.949738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878925.949741:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3912 00000100:00080000:1.0:1788878925.949743:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878925.949745:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878925.949749:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878925.949755:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511560320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878925.949763:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878925/real 1788878925] req@ffffa0618869a680 x1875771511559808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878925.949785:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511559808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878925.949798:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878925.949803:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878925.949807:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511559808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878925.949832:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878925.949855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878925.949870:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878925/real 1788878925] req@ffffa06188699f80 x1875771511559936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878925.949884:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511559936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878925.949893:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878925.949896:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878925.949900:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511559936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878925.949906:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878925.949926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878925.949938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878925/real 1788878925] req@ffffa06188698000 x1875771511560064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878925.949951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511560064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878925.949959:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878925.949962:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878925.949965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878925.949971:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878925.949993:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878925.950004:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878925/real 1788878925] req@ffffa0618869ad80 x1875771511560192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878925.950016:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511560192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878925.950025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878925.950029:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878925.950032:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878925.950038:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878925.950057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878925.950068:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878925/real 1788878925] req@ffffa0618869bb80 x1875771511560320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878925.950080:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511560320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878925.950089:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878925.950092:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878925.950095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878926.589455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878926.589541:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878926.845486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878927.613520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878927.613590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878927.869596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878928.637532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878928.637684:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878928.893532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878929.661609:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878929.661645:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878929.917471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878930.685550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878930.685716:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878930.941524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878931.069590:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878931.069601:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878931.069642:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878931.069655:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3917 00000100:00080000:1.0:1788878931.069667:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878931.069693:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878931.069744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511560448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878931.069804:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878931.069811:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878931.069814:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878931.069822:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3917 00000100:00080000:1.0:1788878931.069845:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878931.069856:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878931.069863:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511560448:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788878931.069888:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878931.069934:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511560576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878931.069954:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878931.069957:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878931.069960:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878931.069964:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3917 00000100:00080000:1.0:1788878931.069968:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878931.069972:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878931.069977:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878931.070005:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511560704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878931.070017:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878931.070022:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878931.070024:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878931.070033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878931.070035:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3917 00000100:00080000:1.0:1788878931.070043:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878931.070048:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878931.070055:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788878931.070070:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511560832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878931.070091:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878931/real 1788878931] req@ffffa0618869aa00 x1875771511560448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878931.070093:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878931.070104:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878931.070109:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878931.070124:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3917 00000100:00080000:1.0:1788878931.070146:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878931.070150:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878931.070155:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878931.070159:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511560448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878931.070168:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511560960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878931.070176:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878931.070186:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878931.070193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511560448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878931.070249:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878931.070293:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878931.070312:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878931/real 1788878931] req@ffffa0618869b800 x1875771511560576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878931.070344:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511560576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878931.070351:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878931.070354:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878931.070357:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878931.070363:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878931.070380:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878931.070388:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878931/real 1788878931] req@ffffa06188699c00 x1875771511560704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878931.070426:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511560704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878931.070435:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878931.070438:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878931.070442:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878931.070448:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878931.070468:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878931.070533:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878931/real 1788878931] req@ffffa06188698380 x1875771511560832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878931.070567:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511560832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878931.070577:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878931.070581:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878931.070584:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878931.070590:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878931.070651:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878931.070663:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878931/real 1788878931] req@ffffa0618869b100 x1875771511560960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878931.070673:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511560960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878931.070704:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878931.070708:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878931.070712:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511560960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878931.709517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878931.709646:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878931.965527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878932.733488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878932.733576:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878932.989535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878933.757502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878933.757623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878934.013520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878934.781476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878934.781611:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878935.037460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878935.805491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878935.805623:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878936.061488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788878936.189504:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878936.189516:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788878936.189522:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878936.189536:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3922 00000100:00080000:1.0:1788878936.189551:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878936.189561:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788878936.189628:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511561088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878936.189673:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878936.189681:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788878936.189695:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878936.189705:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3922 00000100:00080000:1.0:1788878936.189712:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878936.189721:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878936.189737:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878936.189760:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511561088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878936.189761:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511561216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878936.189786:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878936.189790:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788878936.189793:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878936.189798:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3922 00000100:00080000:1.0:1788878936.189802:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878936.189807:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878936.189813:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878936.189846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511561344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878936.189872:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878936.189877:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788878936.189880:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788878936.189886:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878936.189893:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3922 00000100:00080000:1.0:1788878936.189902:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878936.189908:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788878936.189917:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878936.189940:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878936/real 1788878936] req@ffffa0618869b100 x1875771511561088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788878936.189941:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511561472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788878936.190008:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878936.190013:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788878936.190018:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878936.190021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511561088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788878936.190028:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3922 00000100:00080000:1.0:1788878936.190038:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788878936.190050:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788878936.190055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788878936.190059:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878936.190068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878936.190077:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511561088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788878936.190083:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511561600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878936.190121:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878936.190154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878936.190170:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878936/real 1788878936] req@ffffa06188698380 x1875771511561216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878936.190186:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511561216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878936.190208:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878936.190212:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878936.190217:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878936.190241:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878936.190261:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878936.190273:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878936/real 1788878936] req@ffffa06188699c00 x1875771511561344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878936.190285:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511561344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878936.190294:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878936.190297:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878936.190300:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878936.190306:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878936.190322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878936.190331:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878936/real 1788878936] req@ffffa0618869b800 x1875771511561472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878936.190343:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511561472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878936.190372:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878936.190376:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878936.190380:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878936.190387:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878936.190406:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878936.190450:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878936/real 1788878936] req@ffffa0618869aa00 x1875771511561600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878936.190465:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511561600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878936.190475:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878936.190479:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878936.190493:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878936.829524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878936.829661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878937.085488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878937.853475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878937.853529:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878938.109504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878938.877493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878938.877629:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878939.133514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878939.901527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878939.901656:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878940.157535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878940.925501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788878940.925571:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878941.181511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878941.309498:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878941.309507:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878941.309511:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878941.309521:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3928 00000100:00080000:3.0:1788878941.309531:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878941.309537:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878941.309589:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511561728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878941.309610:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878941.309613:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878941.309614:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878941.309617:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3928 00000100:00080000:3.0:1788878941.309619:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878941.309622:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878941.309630:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878941.309640:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511561856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878941.309649:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511561728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878941.309651:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878941.309658:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878941.309662:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878941.309665:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3928 00000100:00080000:3.0:1788878941.309667:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878941.309669:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878941.309673:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878941.309682:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511561984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878941.309688:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878941.309691:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878941.309692:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878941.309695:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3928 00000100:00080000:3.0:1788878941.309698:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878941.309700:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878941.309703:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878941.309710:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511562112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878941.309715:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878941.309718:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878941.309719:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878941.309724:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788878941.309727:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3928 00000100:00080000:3.0:1788878941.309731:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878941.309735:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788878941.309738:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878941.309747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511562240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878941.309762:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878941/real 1788878941] req@ffffa06188871c00 x1875771511561728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878941.309778:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511561728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878941.309786:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878941.309788:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878941.309792:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511561728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878941.309809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878941.309825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878941.309832:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878941/real 1788878941] req@ffffa06188870a80 x1875771511561856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878941.309840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511561856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878941.309846:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878941.309848:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878941.309850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878941.309854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878941.309865:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878941.309871:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878941/real 1788878941] req@ffffa06188870380 x1875771511561984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878941.309882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511561984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878941.309887:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878941.309889:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878941.309891:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511561984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878941.309894:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878941.309904:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878941.309910:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878941/real 1788878941] req@ffffa06188871880 x1875771511562112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878941.309917:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511562112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878941.309922:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878941.309924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878941.309926:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878941.309929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878941.309939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878941.309944:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878941/real 1788878941] req@ffffa06188873800 x1875771511562240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878941.309951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511562240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878941.309964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878941.309968:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878941.309972:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878941.949526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878941.949586:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878942.205493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878942.973488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878942.973570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878943.229443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878943.997484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878943.997595:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878944.253475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878945.021511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878945.021584:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878945.277482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878946.045505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878946.045645:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878946.301461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878946.429629:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878946.429644:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878946.429650:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878946.429664:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3933 00000100:00080000:2.0:1788878946.429679:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878946.429689:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878946.429743:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511562368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878946.429784:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878946.429790:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878946.429792:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878946.429798:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3933 00000100:00080000:2.0:1788878946.429802:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878946.429806:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878946.429819:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878946.429833:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511562496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878946.429847:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878946.429852:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878946.429854:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878946.429859:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3933 00000100:00080000:2.0:1788878946.429864:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878946.429872:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878946.429875:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511562368:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788878946.429879:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878946.429909:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511562624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878946.429923:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878946.429928:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878946.429931:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878946.429936:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3933 00000100:00080000:2.0:1788878946.429942:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878946.429947:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878946.429952:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878946.429964:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511562752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878946.429971:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878946.429987:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878946.429992:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878946.429995:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878946.430004:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3933 00000100:00080000:2.0:1788878946.430012:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878946.430017:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878946/real 1788878946] req@ffffa061888ace00 x1875771511562368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878946.430026:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878946.430044:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878946.430046:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511562368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878946.430068:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878946.430071:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878946.430074:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511562368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878946.430080:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788878946.430105:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511562880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878946.430114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878946.430141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878946/real 1788878946] req@ffffa061888ad180 x1875771511562496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878946.430159:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511562496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878946.430170:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878946.430174:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878946.430180:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878946.430224:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878946.430242:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878946.430250:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878946/real 1788878946] req@ffffa061888ac700 x1875771511562624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878946.430259:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511562624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878946.430264:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878946.430266:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878946.430269:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878946.430273:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878946.430284:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878946.430290:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878946/real 1788878946] req@ffffa061888aed80 x1875771511562752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878946.430298:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511562752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878946.430304:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878946.430306:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878946.430308:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878946.430312:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878946.430322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878946.430328:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878946/real 1788878946] req@ffffa061888ad880 x1875771511562880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878946.430334:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511562880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878946.430340:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878946.430342:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878946.430344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511562880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878947.069481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878947.069596:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878947.325590:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878948.093557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878948.093631:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878948.349629:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878949.117475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878949.117557:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878949.373476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878950.141535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878950.141686:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878950.397557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878951.165599:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878951.165679:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878951.421634:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878951.549491:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878951.549504:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878951.549509:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878951.549521:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3938 00000100:00080000:2.0:1788878951.549533:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878951.549541:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878951.549585:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511563008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878951.549618:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878951.549623:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878951.549626:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878951.549634:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3938 00000100:00080000:2.0:1788878951.549644:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878951.549651:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878951.549664:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878951.549673:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511563008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878951.549684:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511563136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878951.549701:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878951.549705:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878951.549708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878951.549715:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3938 00000100:00080000:2.0:1788878951.549724:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878951.549733:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878951.549741:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878951.549759:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511563264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878951.549768:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788878951.549773:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878951.549775:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878951.549779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878951.549784:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3938 00000100:00080000:2.0:1788878951.549786:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878951.549789:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878951.549793:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878951.549805:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511563392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878951.549812:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878951.549815:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878951.549817:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878951.549820:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878951/real 1788878951] req@ffffa061888ac380 x1875771511563008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878951.549822:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3938 00000100:00080000:2.0:1788878951.549827:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878951.549833:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878951.549839:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788878951.549847:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511563008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878951.549848:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511563520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878951.549881:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878951.549887:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878951.549892:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511563008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878951.549918:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878951.549943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878951.549953:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878951/real 1788878951] req@ffffa061888ae680 x1875771511563136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878951.549962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511563136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878951.549969:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878951.549972:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878951.549975:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878951.549980:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878951.549992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878951.549998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878951/real 1788878951] req@ffffa061888af100 x1875771511563264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878951.550006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511563264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878951.550012:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878951.550014:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878951.550016:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878951.550019:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878951.550031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878951.550038:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878951/real 1788878951] req@ffffa061888ae300 x1875771511563392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878951.550045:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511563392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878951.550051:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878951.550052:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878951.550054:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878951.550058:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878951.550069:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878951.550076:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878951/real 1788878951] req@ffffa061888aea00 x1875771511563520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878951.550083:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511563520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878951.550089:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878951.550091:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878951.550093:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878952.189462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878952.189569:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878952.445571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878953.213457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878953.213532:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878953.469501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878954.237519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878954.237596:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878954.493501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878955.261526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878955.261723:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878955.517523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878956.285462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878956.285533:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878956.541486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878956.669491:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878956.669500:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878956.669504:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878956.669514:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3943 00000100:00080000:2.0:1788878956.669525:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878956.669533:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878956.669568:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511563648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878956.669588:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878956.669591:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878956.669593:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878956.669596:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3943 00000100:00080000:2.0:1788878956.669598:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878956.669600:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878956.669609:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878956.669619:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511563776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878956.669627:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878956.669629:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511563648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878956.669630:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878956.669633:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878956.669639:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3943 00000100:00080000:2.0:1788878956.669641:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878956.669643:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878956.669647:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878956.669654:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511563904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878956.669660:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878956.669663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878956.669664:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878956.669667:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3943 00000100:00080000:2.0:1788878956.669669:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878956.669671:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878956.669674:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878956.669681:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511564032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878956.669687:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878956.669689:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878956.669691:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878956.669693:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3943 00000100:00080000:2.0:1788878956.669695:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878956.669697:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878956.669700:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788878956.669702:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878956.669710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511564160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878956.669739:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878956/real 1788878956] req@ffffa061888afb80 x1875771511563648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878956.669755:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511563648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878956.669763:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878956.669765:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878956.669768:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511563648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878956.669787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878956.669801:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878956.669809:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878956/real 1788878956] req@ffffa061888adc00 x1875771511563776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878956.669816:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511563776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878956.669826:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878956.669828:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878956.669830:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878956.669834:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878956.669845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878956.669850:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878956/real 1788878956] req@ffffa061888ad500 x1875771511563904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878956.669857:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511563904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878956.669862:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878956.669864:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878956.669866:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511563904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878956.669869:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878956.669879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878956.669885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878956/real 1788878956] req@ffffa061888ac000 x1875771511564032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878956.669892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511564032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878956.669897:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878956.669898:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878956.669901:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878956.669904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878956.669913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878956.669918:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878956/real 1788878956] req@ffffa061888adf80 x1875771511564160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878956.669924:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511564160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878956.669929:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878956.669931:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878956.669933:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878957.309481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878957.309550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878957.565572:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878958.333542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878958.333659:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878958.589502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878959.357546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878959.357697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878959.613529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878960.381491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878960.381568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878960.637472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878961.405515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878961.405641:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878961.661540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878961.789480:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878961.789489:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878961.789494:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878961.789504:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3948 00000100:00080000:2.0:1788878961.789521:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878961.789528:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878961.789566:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511564288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878961.789589:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878961.789593:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878961.789594:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878961.789597:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3948 00000100:00080000:2.0:1788878961.789599:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878961.789602:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878961.789611:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878961.789620:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511564416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878961.789627:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878961.789630:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878961.789631:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878961.789633:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3948 00000100:00080000:2.0:1788878961.789635:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878961.789637:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878961.789641:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878961.789649:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511564544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788878961.789661:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511564288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878961.789664:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878961.789671:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878961.789672:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878961.789675:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3948 00000100:00080000:2.0:1788878961.789677:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878961.789680:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878961.789683:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878961.789692:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511564672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878961.789698:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878961.789700:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878961.789702:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878961.789704:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3948 00000100:00080000:2.0:1788878961.789707:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878961.789709:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878961.789712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878961.789719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511564800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788878961.789743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878961.789782:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878961/real 1788878961] req@ffffa061888af480 x1875771511564288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878961.789803:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511564288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878961.789815:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878961.789820:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878961.789824:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511564288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788878961.789832:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878961.789866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878961.789881:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878961/real 1788878961] req@ffffa061888aca80 x1875771511564416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878961.789895:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511564416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878961.789905:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878961.789908:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878961.789912:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878961.789939:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788878961.789960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878961.789971:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878961/real 1788878961] req@ffffa061888adf80 x1875771511564544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878961.789985:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511564544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878961.789994:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878961.789997:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878961.790001:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788878961.790007:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878961.790026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878961.790037:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878961/real 1788878961] req@ffffa061888ac000 x1875771511564672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878961.790050:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511564672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878961.790060:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878961.790063:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878961.790067:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788878961.790073:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788878961.790091:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788878961.790101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878961/real 1788878961] req@ffffa061888ad500 x1875771511564800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878961.790112:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511564800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878961.790121:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878961.790124:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788878961.790128:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511564800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878962.429489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878962.429560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878962.685447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878963.453540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878963.453617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878963.709529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878964.477534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878964.477621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878964.733521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878965.501506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878965.501617:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878965.757532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878966.525576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878966.525834:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878966.781555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788878966.909625:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878966.909646:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788878966.909654:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878966.909674:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3953 00000100:00080000:3.0:1788878966.909695:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878966.909710:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788878966.909782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511564928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878966.909839:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878966.909848:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788878966.909863:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878966.909873:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3953 00000100:00080000:3.0:1788878966.909882:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878966.909897:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878966.909900:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511564928:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788878966.909920:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878966.909965:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511565056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878966.909992:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878966.909999:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788878966.910003:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878966.910012:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3953 00000100:00080000:3.0:1788878966.910021:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878966.910034:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788878966.910037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788878966.910054:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788878966.910086:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511565184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878966.910104:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878966.910114:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878966/real 1788878966] req@ffffa06188872d80 x1875771511564928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878966.910115:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788878966.910122:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878966.910137:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3953 00000100:00080000:3.0:1788878966.910151:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878966.910161:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878966.910171:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511564928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878966.910173:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788878966.910199:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788878966.910202:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511565312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878966.910215:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878966.910233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511564928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788878966.910234:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878966.910254:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788878966.910260:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788878966.910276:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3953 00000100:00080000:3.0:1788878966.910284:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878966.910293:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788878966.910303:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565056:192.168.203.111@tcp:38:kworker.0 00000008:00100000:3.0:1788878966.910322:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878966.910345:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878966.910441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878966/real 1788878966] req@ffffa06188870700 x1875771511565056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878966.910458:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511565440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878966.910467:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511565056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878966.910510:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878966.910518:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878966.910524:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878966.910536:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878966.910568:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878966.910583:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878966/real 1788878966] req@ffffa06188873b80 x1875771511565184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878966.910598:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511565184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878966.910617:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878966.910621:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878966.910625:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878966.910632:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878966.910653:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878966.910665:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878966/real 1788878966] req@ffffa06188872300 x1875771511565312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878966.910679:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511565312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878966.910689:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878966.910693:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878966.910696:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878966.910718:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878966.910746:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878966.910759:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878966/real 1788878966] req@ffffa06188872680 x1875771511565440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878966.910773:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511565440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878966.910783:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878966.910786:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878966.910791:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878967.549649:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878967.549709:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878967.805556:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878968.573548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878968.573657:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878968.829531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878969.597598:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878969.597663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878969.853500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878970.621492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878970.621563:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878970.877485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878971.645447:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878971.645528:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878971.901537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878972.029629:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878972.029646:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878972.029653:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878972.029669:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3958 00000100:00080000:2.0:1788878972.029685:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878972.029716:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878972.029782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511565568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878972.029837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878972.029846:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878972.029850:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878972.029859:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3958 00000100:00080000:2.0:1788878972.029868:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878972.029879:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878972.029898:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878972.029909:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511565568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878972.029949:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511565696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878972.029967:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878972.029973:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878972.029976:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878972.029982:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3958 00000100:00080000:2.0:1788878972.029988:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878972.029994:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878972.030002:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878972.030021:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511565824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878972.030044:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565696:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878972.030048:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878972.030055:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878972.030058:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878972.030066:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3958 00000100:00080000:2.0:1788878972.030073:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878972.030077:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565824:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788878972.030084:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878972.030093:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878972.030135:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511565952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878972.030168:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878972.030175:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878972.030178:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878972.030187:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3958 00000100:00100000:0.0:1788878972.030203:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565952:192.168.203.111@tcp:8:kworker.0 00000100:00080000:2.0:1788878972.030211:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878972.030223:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878972.030233:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878972.030260:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511566080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878972.030325:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878972.669509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878972.669730:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878972.925489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878973.693475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878973.693564:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878973.949548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878974.717565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878974.717646:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878974.973505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788878975.101796:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788878975.101854:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788878975.101880:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878975.101980:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878975.101993:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878975.102009:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788878975.102023:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788878975.102091:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878972/real 1788878975] req@ffffa061888af100 x1875771511566080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878975.102156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511566080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878975.102193:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878975.102201:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878975.102208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878975.102223:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878972/real 1788878975] req@ffffa061888ae300 x1875771511565952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878975.102246:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511565952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878975.102261:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878975.102267:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878975.102273:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878975.102285:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878972/real 1788878975] req@ffffa061888aea00 x1875771511565824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878975.102307:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511565824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878975.102346:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878975.102353:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878975.102359:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878975.102372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878972/real 1788878975] req@ffffa061888afb80 x1875771511565696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878975.102439:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511565696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878975.102456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878975.102460:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878975.102465:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511565696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878975.102478:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878972/real 1788878975] req@ffffa061888adc00 x1875771511565568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878975.102502:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511565568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878975.102515:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878975.102520:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878975.102525:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511565568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878975.741504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878975.741626:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878975.997515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878976.765516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878976.765600:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878977.021509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878977.149526:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878977.149538:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878977.149544:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878977.149555:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3963 00000100:00080000:2.0:1788878977.149567:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878977.149576:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878977.149618:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511566208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878977.149639:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878977.149643:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878977.149645:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878977.149649:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3963 00000100:00080000:2.0:1788878977.149652:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878977.149655:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878977.149665:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878977.149667:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511566208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878977.149676:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511566336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878977.149686:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878977.149704:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878977.149705:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878977.149708:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3963 00000100:00080000:2.0:1788878977.149711:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878977.149713:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878977.149720:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878977.149732:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511566464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878977.149752:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878977.149756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878977.149758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878977.149761:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3963 00000100:00080000:2.0:1788878977.149766:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878977.149769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878977.149774:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878977.149781:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878977.149794:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511566592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878977.149805:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878977.149809:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878977.149811:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878977.149817:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3963 00000100:00100000:0.0:1788878977.149818:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878977/real 1788878977] req@ffffa061888ae680 x1875771511566208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878977.149821:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878977.149830:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878977.149848:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878977.149860:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511566720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878977.149867:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511566208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878977.149881:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878977.149885:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878977.149892:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511566208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878977.149923:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878977.149975:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878977.149991:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878977/real 1788878977] req@ffffa061888ac380 x1875771511566336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878977.150005:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511566336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878977.150015:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878977.150019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878977.150023:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878977.150029:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878977.150068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878977.150079:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878977/real 1788878977] req@ffffa061888ad880 x1875771511566464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878977.150091:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511566464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878977.150100:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878977.150119:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878977.150122:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878977.150128:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878977.150148:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878977.150159:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878977/real 1788878977] req@ffffa061888aed80 x1875771511566592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878977.150173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511566592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878977.150181:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878977.150185:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878977.150189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878977.150195:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878977.150217:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878977.150229:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878977/real 1788878977] req@ffffa061888ac700 x1875771511566720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878977.150241:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511566720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878977.150250:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878977.150252:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878977.150255:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878977.789483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878977.789543:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878978.045542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878978.813524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878978.813609:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878979.069545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878979.837511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878979.837620:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878980.093494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878980.861478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878980.861568:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878981.117492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878981.885467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878981.885531:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878982.141448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878982.269464:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878982.269472:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878982.269477:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878982.269487:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3968 00000100:00080000:2.0:1788878982.269497:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878982.269504:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878982.269538:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511566848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878982.269561:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878982.269564:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878982.269566:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878982.269569:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3968 00000100:00080000:2.0:1788878982.269571:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878982.269574:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878982.269584:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878982.269597:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511566848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878982.269597:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511566976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878982.269612:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878982.269615:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878982.269616:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878982.269619:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3968 00000100:00080000:2.0:1788878982.269622:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878982.269625:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878982.269629:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878982.269639:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511567104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878982.269646:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878982.269649:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878982.269650:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878982.269652:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3968 00000100:00080000:2.0:1788878982.269654:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878982.269656:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878982.269660:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878982.269672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878982.269672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511567232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878982.269682:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878982.269685:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878982.269686:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878982.269690:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3968 00000100:00080000:2.0:1788878982.269692:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878982.269695:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878982.269697:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878982.269705:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511567360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878982.269712:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878982/real 1788878982] req@ffffa061888ad180 x1875771511566848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878982.269729:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511566848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878982.269738:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878982.269741:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878982.269745:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511566848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878982.269764:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878982.269780:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878982.269788:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878982/real 1788878982] req@ffffa061888ace00 x1875771511566976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878982.269796:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511566976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878982.269802:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878982.269804:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878982.269806:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511566976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878982.269810:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878982.269819:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878982.269826:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878982/real 1788878982] req@ffffa061888ac700 x1875771511567104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878982.269838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511567104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878982.269847:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878982.269850:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878982.269854:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878982.269860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878982.269882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878982.269893:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878982/real 1788878982] req@ffffa061888aed80 x1875771511567232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878982.269901:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511567232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878982.269907:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878982.269909:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878982.269911:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878982.269915:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878982.269926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878982.269933:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878982/real 1788878982] req@ffffa061888ad880 x1875771511567360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878982.269940:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511567360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878982.269945:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878982.269947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878982.269949:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878982.909535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878982.909561:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878983.165489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878983.933584:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878983.933659:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878984.189519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878984.957540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878984.957607:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878985.213474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878985.981515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878985.981673:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878986.237506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878987.005478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878987.005547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878987.261493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878987.389559:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878987.389576:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878987.389585:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878987.389601:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3974 00000100:00080000:2.0:1788878987.389621:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878987.389633:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878987.389693:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511567488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878987.389732:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878987.389737:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878987.389740:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878987.389747:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3974 00000100:00080000:2.0:1788878987.389751:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878987.389757:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878987.389772:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878987.389793:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511567616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878987.389797:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511567488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878987.389841:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878987.389846:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878987.389849:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878987.389856:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3974 00000100:00080000:2.0:1788878987.389862:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878987.389867:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878987.389875:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878987.389892:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511567744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878987.389905:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878987.389910:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878987.389916:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878987.389918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878987.389927:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3974 00000100:00080000:2.0:1788878987.389934:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878987.389939:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878987.389947:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878987.389963:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511567872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878987.389976:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878987.389986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878987/real 1788878987] req@ffffa061888ac380 x1875771511567488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878987.389987:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878987.389995:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878987.390010:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3974 00000100:00080000:2.0:1788878987.390019:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878987.390028:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878987.390035:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788878987.390043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511567488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878987.390053:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511568000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878987.390074:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878987.390089:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878987.390101:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511567488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878987.390152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878987.390183:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878987.390200:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878987/real 1788878987] req@ffffa061888ae680 x1875771511567616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878987.390216:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511567616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878987.390259:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878987.390265:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878987.390270:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878987.390279:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878987.390312:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878987.390335:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878987/real 1788878987] req@ffffa061888adc00 x1875771511567744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878987.390362:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511567744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878987.390429:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878987.390435:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878987.390441:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878987.390454:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878987.390499:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878987.390525:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878987/real 1788878987] req@ffffa061888afb80 x1875771511567872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878987.390551:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511567872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878987.390562:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878987.390566:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878987.390570:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511567872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878987.390578:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878987.390606:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878987.390620:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878987/real 1788878987] req@ffffa061888aea00 x1875771511568000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878987.390635:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511568000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878987.390644:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878987.390648:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878987.390654:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878988.029539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878988.029615:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878988.285504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878989.053518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878989.053631:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878989.309517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878990.077564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878990.077606:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878990.333506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878991.101546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878991.101643:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878991.357527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878992.125507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878992.125516:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878992.381474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878992.509570:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878992.509596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878992.509603:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878992.509618:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3979 00000100:00080000:2.0:1788878992.509635:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878992.509646:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878992.509720:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511568128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878992.509771:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878992.509777:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878992.509780:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878992.509789:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3979 00000100:00080000:2.0:1788878992.509794:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878992.509799:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878992.509825:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878992.509839:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511568128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788878992.509856:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511568256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878992.509875:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878992.509882:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878992.509886:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878992.509892:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3979 00000100:00080000:2.0:1788878992.509896:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878992.509902:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878992.509912:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878992.509930:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511568384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878992.509943:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878992.509949:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878992.509951:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878992.509955:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3979 00000100:00080000:2.0:1788878992.509959:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878992.509964:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878992.509979:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788878992.509981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788878992.510001:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511568512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878992.510018:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878992.510024:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878992.510027:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878992.510034:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3979 00000100:00080000:2.0:1788878992.510038:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878992.510045:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878992.510055:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878992.510083:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511568640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878992.510089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878992/real 1788878992] req@ffffa061888ae300 x1875771511568128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878992.510169:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511568128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878992.510186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878992.510238:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878992.510241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511568128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878992.510266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878992.510308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878992.510326:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878992/real 1788878992] req@ffffa061888af100 x1875771511568256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878992.510343:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511568256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878992.510416:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878992.510422:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878992.510430:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878992.510462:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878992.510481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878992.510489:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878992/real 1788878992] req@ffffa061888ad500 x1875771511568384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878992.510498:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511568384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878992.510505:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878992.510508:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878992.510528:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878992.510537:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878992.510560:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878992.510572:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878992/real 1788878992] req@ffffa061888ac000 x1875771511568512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878992.510598:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511568512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878992.510604:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878992.510606:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878992.510608:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878992.510613:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878992.510625:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878992.510632:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878992/real 1788878992] req@ffffa061888adf80 x1875771511568640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878992.510640:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511568640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878992.510665:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878992.510669:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878992.510673:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788878993.149569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878993.149654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878993.405462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878994.173548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878994.173667:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878994.429592:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878995.197506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878995.197664:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878995.453518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788878996.221498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878996.221663:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878996.477523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878997.245539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878997.245644:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878997.501499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788878997.629504:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878997.629516:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788878997.629522:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878997.629536:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3984 00000100:00080000:2.0:1788878997.629551:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878997.629560:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788878997.629616:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511568768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878997.629644:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878997.629648:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788878997.629649:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878997.629653:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3984 00000100:00080000:2.0:1788878997.629657:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878997.629660:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878997.629682:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878997.629696:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511568896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878997.629713:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511568768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788878997.629713:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878997.629734:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788878997.629738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878997.629742:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3984 00000100:00080000:2.0:1788878997.629744:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878997.629747:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878997.629752:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878997.629767:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511569024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878997.629776:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878997.629780:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788878997.629781:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878997.629785:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3984 00000100:00080000:2.0:1788878997.629788:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788878997.629791:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878997.629796:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878997.629804:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511569152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878997.629812:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878997.629816:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788878997.629827:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878997.629830:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3984 00000100:00080000:2.0:1788878997.629833:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878997.629834:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788878997.629839:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788878997.629845:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788878997.629857:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511569280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878997.629890:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878997/real 1788878997] req@ffffa061888aca80 x1875771511568768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878997.629907:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511568768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878997.629916:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878997.629920:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878997.629923:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511568768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788878997.629945:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878997.629966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878997.629979:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878997/real 1788878997] req@ffffa061888af480 x1875771511568896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878997.630004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511568896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878997.630015:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878997.630019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878997.630023:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511568896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878997.630031:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788878997.630050:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878997.630059:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878997/real 1788878997] req@ffffa061888adf80 x1875771511569024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878997.630067:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511569024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878997.630073:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878997.630076:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878997.630078:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788878997.630082:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878997.630095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878997.630101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878997/real 1788878997] req@ffffa061888ac000 x1875771511569152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878997.630115:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511569152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878997.630121:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878997.630123:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878997.630125:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878997.630129:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788878997.630140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788878997.630170:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878997/real 1788878997] req@ffffa061888ad500 x1875771511569280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878997.630179:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511569280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878997.630205:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878997.630208:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788878997.630211:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788878998.269506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788878998.269540:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878998.525498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788878999.293505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788878999.293657:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788878999.549526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879000.317542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879000.317691:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879000.573497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879001.341505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879001.341575:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879001.597499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879002.365443:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879002.365553:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879002.621459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879002.749500:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879002.749509:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879002.749514:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879002.749524:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3989 00000100:00080000:1.0:1788879002.749535:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879002.749542:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879002.749581:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511569408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879002.749605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879002.749608:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879002.749609:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879002.749612:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3989 00000100:00080000:1.0:1788879002.749615:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879002.749617:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879002.749625:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879002.749635:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511569536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879002.749643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511569408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879002.749646:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879002.749652:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879002.749654:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879002.749657:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3989 00000100:00080000:1.0:1788879002.749659:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879002.749661:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879002.749664:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879002.749672:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511569664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879002.749679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879002.749682:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879002.749683:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879002.749685:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3989 00000100:00080000:1.0:1788879002.749687:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879002.749690:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879002.749693:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879002.749699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511569792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879002.749704:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879002.749707:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879002.749708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879002.749711:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3989 00000100:00080000:1.0:1788879002.749713:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879002.749715:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879002.749718:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879002.749722:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879002.749729:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511569920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879002.749765:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879002/real 1788879002] req@ffffa0618869bb80 x1875771511569408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879002.749782:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511569408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879002.749791:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879002.749794:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879002.749797:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511569408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879002.749818:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879002.749833:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879002.749841:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879002/real 1788879002] req@ffffa0618869ad80 x1875771511569536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879002.749849:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511569536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879002.749855:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879002.749858:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879002.749861:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879002.749864:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879002.749875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879002.749881:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879002/real 1788879002] req@ffffa06188698000 x1875771511569664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879002.749889:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511569664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879002.749894:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879002.749896:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879002.749898:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879002.749902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879002.749914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879002.749920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879002/real 1788879002] req@ffffa06188699f80 x1875771511569792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879002.749928:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511569792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879002.749933:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879002.749935:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879002.749937:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879002.749940:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879002.749949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879002.749955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879002/real 1788879002] req@ffffa0618869a680 x1875771511569920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879002.749962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511569920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879002.749972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879002.749974:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879002.749976:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511569920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879003.389465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879003.389530:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879003.645545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879004.413498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879004.413579:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879004.669473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879005.437508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879005.437587:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879005.693508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879006.461532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879006.461658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879006.717516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879007.485541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879007.485650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879007.741489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879007.869535:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879007.869550:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879007.869557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879007.869571:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3994 00000100:00080000:1.0:1788879007.869586:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879007.869598:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879007.869651:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511570048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879007.869691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879007.869696:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879007.869699:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879007.869704:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3994 00000100:00080000:1.0:1788879007.869708:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879007.869713:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879007.869726:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879007.869745:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511570176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879007.869757:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511570048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879007.869769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879007.869776:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879007.869779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879007.869786:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3994 00000100:00080000:1.0:1788879007.869792:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879007.869805:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879007.869811:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879007.869836:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511570304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879007.869856:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879007.869864:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879007.869868:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879007.869871:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879007.869886:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3994 00000100:00080000:1.0:1788879007.869899:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879007.869909:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879007.869922:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879007.869930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879007/real 1788879007] req@ffffa0618869a680 x1875771511570048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879007.869960:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511570048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879007.869974:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511570432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879007.869978:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879007.869994:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788879007.870010:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879007.870013:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511570048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879007.870018:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879007.870025:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879007.870037:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3994 00000100:00080000:1.0:1788879007.870043:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879007.870050:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879007.870057:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879007.870079:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879007.870130:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511570560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879007.870176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879007.870200:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879007/real 1788879007] req@ffffa06188699f80 x1875771511570176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879007.870217:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511570176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879007.870228:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879007.870235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879007.870240:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879007.870249:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879007.870269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879007.870280:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879007/real 1788879007] req@ffffa06188698000 x1875771511570304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879007.870294:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511570304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879007.870303:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879007.870307:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879007.870311:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879007.870317:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879007.870334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879007.870344:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879007/real 1788879007] req@ffffa0618869ad80 x1875771511570432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879007.870357:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511570432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879007.870367:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879007.870380:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879007.870414:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879007.870436:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879007.870458:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879007.870471:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879007/real 1788879007] req@ffffa0618869bb80 x1875771511570560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879007.870484:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511570560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879007.870504:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879007.870508:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879007.870511:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879008.509527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879008.509702:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879008.765518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879009.533471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879009.533661:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879009.789529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879010.557505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879010.557586:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879010.813470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879011.581536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879011.581649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879011.837516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879012.605524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879012.605666:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879012.861504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879012.989551:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879012.989572:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879012.989577:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879012.989590:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 3999 00000100:00080000:2.0:1788879012.989604:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879012.989613:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879012.989649:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511570688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879012.989675:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879012.989678:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879012.989680:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879012.989684:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3999 00000100:00080000:2.0:1788879012.989688:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879012.989694:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879012.989705:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879012.989726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511570816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879012.989734:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511570688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879012.989749:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879012.989756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879012.989758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879012.989765:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3999 00000100:00080000:2.0:1788879012.989771:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879012.989777:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879012.989785:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879012.989807:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511570944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879012.989821:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879012.989825:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879012.989827:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879012.989835:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3999 00000800:00000100:0.0:1788879012.989836:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879012.989847:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879012.989858:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879012.989866:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879012.989887:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511571072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879012.989891:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879012/real 1788879012] req@ffffa061888af100 x1875771511570688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879012.989928:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511570688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879012.989928:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879012.989956:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879012.989962:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879012.989965:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879012.989973:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788879012.989973:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 3999 00000100:00100000:0.0:1788879012.989987:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511570688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879012.989990:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879012.990001:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879012.990010:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879012.990020:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879012.990032:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511571200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879012.990047:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879012.990094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879012/real 1788879012] req@ffffa061888ae300 x1875771511570816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879012.990126:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511570816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879012.990140:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879012.990144:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879012.990147:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879012.990155:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879012.990190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879012.990201:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879012/real 1788879012] req@ffffa061888aea00 x1875771511570944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879012.990219:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511570944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879012.990227:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879012.990229:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879012.990232:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511570944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879012.990238:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879012.990255:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879012.990264:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879012/real 1788879012] req@ffffa061888afb80 x1875771511571072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879012.990275:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511571072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879012.990283:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879012.990285:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879012.990288:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879012.990316:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879012.990331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879012.990340:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879012/real 1788879012] req@ffffa061888adc00 x1875771511571200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879012.990351:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511571200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879012.990365:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879012.990377:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879012.990407:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879013.629529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879013.629616:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879013.885530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879014.653523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879014.653649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879014.909505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879015.677526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879015.677608:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879015.933508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879016.701480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879016.701604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879016.957519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879017.725463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879017.725561:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879017.981464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879018.109482:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879018.109491:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879018.109496:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879018.109507:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4004 00000100:00080000:2.0:1788879018.109518:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879018.109538:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879018.109587:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511571328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879018.109623:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879018.109627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879018.109629:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879018.109633:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4004 00000100:00080000:2.0:1788879018.109637:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879018.109644:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879018.109655:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879018.109666:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511571456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879018.109676:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879018.109679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788879018.109689:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511571328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879018.109692:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879018.109701:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4004 00000100:00080000:2.0:1788879018.109705:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879018.109707:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879018.109712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879018.109748:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511571584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879018.109759:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879018.109764:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879018.109766:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879018.109771:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4004 00000100:00080000:2.0:1788879018.109786:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879018.109791:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879018.109797:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788879018.109806:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879018.109814:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511571712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879018.109827:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879018.109831:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879018.109834:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879018.109838:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4004 00000100:00080000:2.0:1788879018.109844:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879018.109849:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879018.109854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879018.109873:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879018/real 1788879018] req@ffffa061888ae680 x1875771511571328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879018.109881:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511571840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879018.109944:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511571328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879018.109970:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879018.109976:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879018.109981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511571328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879018.110057:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879018.110080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879018.110091:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879018/real 1788879018] req@ffffa061888ac380 x1875771511571456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879018.110101:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511571456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879018.110108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879018.110111:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879018.110113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879018.110118:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879018.110131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879018.110138:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879018/real 1788879018] req@ffffa061888ad880 x1875771511571584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879018.110146:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511571584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879018.110152:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879018.110154:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879018.110156:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879018.110160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879018.110172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879018.110178:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879018/real 1788879018] req@ffffa061888aed80 x1875771511571712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879018.110192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511571712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879018.110198:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879018.110200:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879018.110202:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879018.110206:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879018.110217:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879018.110223:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879018/real 1788879018] req@ffffa061888ac700 x1875771511571840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879018.110230:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511571840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879018.110236:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879018.110238:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879018.110241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511571840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879018.749511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879018.749671:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879019.005542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879019.773523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879019.773701:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879020.029531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879020.797531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879020.797608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879021.053530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879021.821500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879021.821662:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879022.077566:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879022.845508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879022.845632:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879023.101531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879023.229525:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879023.229541:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879023.229548:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879023.229567:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4009 00000100:00080000:1.0:1788879023.229586:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879023.229601:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879023.229673:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511571968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879023.229712:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879023.229718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879023.229722:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879023.229729:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4009 00000100:00080000:1.0:1788879023.229735:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879023.229741:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879023.229754:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879023.229771:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511572096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879023.229786:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879023.229791:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879023.229794:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879023.229799:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4009 00000100:00080000:1.0:1788879023.229803:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879023.229808:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879023.229814:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879023.229826:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511572224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879023.229843:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511571968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879023.229845:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879023.229860:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879023.229865:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879023.229872:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4009 00000100:00080000:1.0:1788879023.229876:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879023.229880:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879023.229887:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879023.229904:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511572352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879023.229918:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879023.229922:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879023.229924:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879023.229928:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4009 00000100:00080000:1.0:1788879023.229944:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879023.229951:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:3.0:1788879023.229961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879023.229964:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879023.230008:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511572480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879023.230055:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879023/real 1788879023] req@ffffa0618869aa00 x1875771511571968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879023.230084:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511571968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879023.230107:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879023.230113:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879023.230118:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511571968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879023.230127:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879023.230158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879023.230173:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879023/real 1788879023] req@ffffa0618869b800 x1875771511572096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879023.230188:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511572096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879023.230198:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879023.230202:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879023.230207:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879023.230238:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879023.230258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879023.230270:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879023/real 1788879023] req@ffffa06188699c00 x1875771511572224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879023.230284:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511572224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879023.230294:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879023.230298:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879023.230301:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879023.230309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879023.230328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879023.230339:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879023/real 1788879023] req@ffffa06188698380 x1875771511572352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879023.230353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511572352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879023.230363:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879023.230366:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879023.230370:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879023.230376:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879023.230440:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879023.230455:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879023/real 1788879023] req@ffffa0618869b100 x1875771511572480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879023.230470:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511572480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879023.230480:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879023.230484:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879023.230488:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879023.869537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879023.869626:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879024.125533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879024.893537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879024.893704:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879025.149510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879025.917550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879025.917648:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879026.173559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879026.941586:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879026.941669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879027.197480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788879027.965656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879027.965736:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:0.0:1788879028.221447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879028.349544:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879028.349559:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879028.349566:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879028.349596:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4015 00000100:00080000:1.0:1788879028.349614:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879028.349626:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879028.349690:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511572608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879028.349729:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879028.349734:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879028.349737:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879028.349743:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4015 00000100:00080000:1.0:1788879028.349748:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879028.349753:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879028.349767:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879028.349783:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511572736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879028.349797:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879028.349801:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879028.349804:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879028.349808:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4015 00000100:00080000:1.0:1788879028.349813:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879028.349828:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879028.349834:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879028.349849:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511572864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879028.349860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511572608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879028.349869:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879028.349876:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879028.349879:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879028.349884:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4015 00000100:00080000:1.0:1788879028.349888:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879028.349892:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879028.349899:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879028.349916:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511572992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879028.349929:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879028.349934:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879028.349937:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879028.349968:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4015 00000100:00080000:1.0:1788879028.349988:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879028.350007:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879028.350018:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788879028.350020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879028.350054:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511573120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879028.350104:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879028/real 1788879028] req@ffffa0618869b100 x1875771511572608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879028.350136:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511572608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879028.350168:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879028.350174:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879028.350180:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511572608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879028.350191:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879028.350224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879028.350240:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879028/real 1788879028] req@ffffa06188698380 x1875771511572736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879028.350256:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511572736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879028.350267:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879028.350271:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879028.350275:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879028.350311:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879028.350347:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879028.350368:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879028/real 1788879028] req@ffffa06188699c00 x1875771511572864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879028.350447:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511572864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879028.350475:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879028.350482:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879028.350489:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879028.350502:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879028.350543:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879028.350565:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879028/real 1788879028] req@ffffa0618869b800 x1875771511572992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879028.350590:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511572992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879028.350606:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879028.350612:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879028.350617:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511572992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879028.350628:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879028.350666:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879028.350689:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879028/real 1788879028] req@ffffa0618869aa00 x1875771511573120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879028.350711:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511573120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879028.350722:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879028.350726:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879028.350731:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879028.989555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879028.989674:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879029.245510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879030.013562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879030.013681:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879030.269511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879031.037532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879031.037705:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879031.293457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879032.061576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879032.061700:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879032.317536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879033.085478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879033.085535:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879033.341492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879033.469521:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879033.469536:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879033.469543:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879033.469559:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4020 00000100:00080000:1.0:1788879033.469574:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879033.469585:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879033.469638:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511573248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879033.469680:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879033.469686:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879033.469689:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879033.469696:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4020 00000100:00080000:1.0:1788879033.469702:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879033.469710:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879033.469725:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879033.469744:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511573248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879033.469748:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511573376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879033.469770:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879033.469776:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879033.469779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879033.469785:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4020 00000100:00080000:1.0:1788879033.469791:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879033.469798:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879033.469806:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879033.469823:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511573504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788879033.469841:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879033.469843:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879033.469852:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879033.469854:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879033.469862:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4020 00000100:00080000:1.0:1788879033.469869:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879033.469876:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879033.469885:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879033.469904:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879033/real 1788879033] req@ffffa0618869bb80 x1875771511573248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879033.469926:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511573632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879033.469961:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788879033.469965:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511573248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879033.469969:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879033.469976:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879033.469987:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879033.469990:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4020 00000100:00080000:3.0:1788879033.469998:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788879033.470001:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788879033.470006:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511573248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879033.470013:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879033.470034:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879033.470051:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879033.470057:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511573760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788879033.470088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879033.470113:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879033/real 1788879033] req@ffffa0618869ad80 x1875771511573376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879033.470127:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511573376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879033.470135:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879033.470138:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879033.470142:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879033.470147:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879033.470174:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879033.470184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879033/real 1788879033] req@ffffa06188698000 x1875771511573504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879033.470195:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511573504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879033.470202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879033.470205:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879033.470208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879033.470214:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879033.470229:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879033.470237:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879033/real 1788879033] req@ffffa06188699f80 x1875771511573632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879033.470248:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511573632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879033.470255:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879033.470258:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879033.470261:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879033.470276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879033.470290:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879033.470299:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879033/real 1788879033] req@ffffa0618869a680 x1875771511573760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879033.470309:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511573760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879033.470317:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879033.470320:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879033.470324:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511573760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879034.109494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879034.109565:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879034.365505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879035.133540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879035.133659:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879035.389511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879036.157482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879036.157642:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879036.413508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879037.181538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879037.181679:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879037.437500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879038.205491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879038.205666:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879038.461515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879038.589530:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879038.589544:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879038.589551:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879038.589568:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4025 00000100:00080000:1.0:1788879038.589588:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879038.589599:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879038.589665:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511573888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879038.589708:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879038.589713:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879038.589715:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879038.589724:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4025 00000100:00080000:1.0:1788879038.589729:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879038.589734:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879038.589748:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879038.589765:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511574016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879038.589781:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879038.589786:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879038.589788:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879038.589793:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4025 00000100:00080000:1.0:1788879038.589798:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879038.589802:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879038.589809:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879038.589815:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511573888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879038.589827:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511574144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879038.589848:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879038.589855:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879038.589860:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879038.589868:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4025 00000100:00080000:1.0:1788879038.589898:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879038.589908:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879038.589921:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879038.589952:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511574272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879038.589974:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879038.589979:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879038.589982:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879038.589989:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4025 00000100:00080000:1.0:1788879038.589997:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788879038.590005:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574016:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879038.590008:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879038.590029:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879038.590053:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511574400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879038.590086:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879038.590121:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879038.590152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879039.229494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879039.229609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879039.485493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879040.253463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879040.253651:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879040.509495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879041.277536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879041.277632:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879041.533496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879041.661840:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879041.661950:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879041.661977:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879041.662050:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879041.662071:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879041.662088:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879041.662106:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788879041.662170:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879038/real 1788879041] req@ffffa06188698000 x1875771511574144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879041.662224:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511574144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879041.662249:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879041.662260:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879041.662270:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879041.662288:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879038/real 1788879041] req@ffffa0618869ad80 x1875771511574272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879041.662315:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511574272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879041.662344:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879041.662362:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879041.662371:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879041.662436:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879038/real 1788879041] req@ffffa0618869bb80 x1875771511574400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879041.662470:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511574400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879041.662487:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879041.662494:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879041.662500:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879041.662512:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879038/real 1788879041] req@ffffa0618869a680 x1875771511573888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879041.662537:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511573888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879041.662552:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879041.662559:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879041.662566:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511573888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879041.662581:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879038/real 1788879041] req@ffffa06188699f80 x1875771511574016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879041.662605:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511574016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879041.662623:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879041.662630:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879041.662636:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879042.301566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879042.301641:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879042.557525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879043.325528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879043.325641:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879043.581534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879043.709639:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879043.709656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879043.709664:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879043.709683:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4030 00000100:00080000:2.0:1788879043.709703:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879043.709718:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879043.709780:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511574528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879043.709829:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879043.709835:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879043.709863:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879043.709872:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4030 00000100:00080000:2.0:1788879043.709880:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879043.709889:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879043.709908:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879043.709940:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511574656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879043.709958:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511574528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879043.709973:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879043.709982:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879043.709986:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879043.709996:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4030 00000100:00080000:2.0:1788879043.710005:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879043.710014:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879043.710025:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879043.710056:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511574784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879043.710074:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879043.710083:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879043.710089:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788879043.710094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879043.710103:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4030 00000100:00080000:2.0:1788879043.710143:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879043.710155:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879043.710167:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879043.710173:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879043/real 1788879043] req@ffffa061888ace00 x1875771511574528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879043.710197:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511574912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879043.710223:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788879043.710225:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511574528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879043.710238:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879043.710248:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879043.710267:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4030 00000100:00080000:3.0:1788879043.710270:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879043.710280:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879043.710286:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788879043.710297:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879043.710305:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511574528:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788879043.710315:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879043.710347:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511575040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879043.710363:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879043.710516:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879043.710547:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879043/real 1788879043] req@ffffa061888ad180 x1875771511574656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879043.710576:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511574656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879043.710595:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879043.710602:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879043.710610:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879043.710625:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879043.710670:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879043.710710:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879043/real 1788879043] req@ffffa061888ac700 x1875771511574784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879043.710737:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511574784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879043.710754:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879043.710761:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879043.710768:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879043.710780:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879043.710825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879043.712186:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879043/real 1788879043] req@ffffa061888aed80 x1875771511574912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879043.712217:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511574912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879043.712231:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879043.712237:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879043.712242:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511574912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879043.712289:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879043.712352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879043.712415:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879043/real 1788879043] req@ffffa061888ad880 x1875771511575040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879043.712448:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511575040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879043.712465:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879043.712473:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879043.712479:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879044.349500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879044.349613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879044.605499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879045.373460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879045.373530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879045.629478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879046.397522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879046.397657:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879046.653464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879047.421495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879047.421570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879047.677537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879048.445562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879048.445719:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879048.701495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879048.829587:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879048.829600:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879048.829607:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879048.829623:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4035 00000100:00080000:3.0:1788879048.829649:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879048.829677:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879048.829731:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511575168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879048.829769:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879048.829774:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879048.829777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879048.829782:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4035 00000100:00080000:3.0:1788879048.829787:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879048.829793:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879048.829861:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511575168:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879048.829866:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879048.829886:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511575296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879048.829911:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879048.829917:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879048.829920:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879048.829926:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4035 00000100:00080000:3.0:1788879048.829933:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879048.829942:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879048.829948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788879048.829956:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879048.829979:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511575424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879048.829992:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879048.829998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879048/real 1788879048] req@ffffa06188872a00 x1875771511575168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879048.829998:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879048.830004:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879048.830015:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4035 00000100:00080000:3.0:1788879048.830025:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879048.830031:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879048.830037:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879048.830040:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511575168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879048.830053:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511575552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879048.830056:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879048.830071:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788879048.830075:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879048.830080:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511575168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879048.830083:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879048.830087:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879048.830094:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4035 00000100:00080000:3.0:1788879048.830099:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879048.830102:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879048.830106:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879048.830117:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879048.830122:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511575680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879048.830151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879048.830168:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879048/real 1788879048] req@ffffa06188873480 x1875771511575296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879048.830180:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511575296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879048.830189:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879048.830192:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879048.830196:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879048.830201:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879048.830216:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879048.830225:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879048/real 1788879048] req@ffffa06188871500 x1875771511575424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879048.830235:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511575424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879048.830243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879048.830245:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879048.830248:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879048.830253:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879048.830270:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879048.830279:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879048/real 1788879048] req@ffffa06188873100 x1875771511575552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879048.830289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511575552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879048.830296:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879048.830299:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879048.830302:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879048.830316:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879048.830331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879048.830340:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879048/real 1788879048] req@ffffa06188872a00 x1875771511575680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879048.830350:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511575680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879048.830358:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879048.830360:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879048.830363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879049.469512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879049.469684:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879049.725528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879050.493533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879050.493623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879050.749505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879051.517489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879051.517759:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879051.773541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879052.541483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879052.541523:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879052.797463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879053.565502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879053.565677:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879053.821511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879053.949617:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879053.949631:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879053.949639:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879053.949654:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4040 00000100:00080000:1.0:1788879053.949669:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879053.949682:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879053.949740:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511575808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879053.949807:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879053.949824:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879053.949827:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879053.949833:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4040 00000100:00080000:1.0:1788879053.949839:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879053.949844:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879053.949859:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879053.949881:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511575936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879053.949898:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879053.949905:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879053.949908:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879053.949925:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511575808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879053.949926:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4040 00000100:00080000:1.0:1788879053.949942:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879053.949950:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879053.949957:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879053.949990:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511576064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879053.950010:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879053.950018:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879053.950022:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879053.950031:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4040 00000100:00080000:1.0:1788879053.950065:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879053.950076:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879053.950084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879053.950098:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879053.950124:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511576192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879053.950143:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879053.950147:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879053.950150:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879053.950161:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4040 00000100:00100000:0.0:1788879053.950162:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879053/real 1788879053] req@ffffa0618869aa00 x1875771511575808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879053.950175:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879053.950193:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879053.950206:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879053.950227:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511576320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879053.950291:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511575808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879053.950311:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879053.950319:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879053.950329:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511575808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879053.950371:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879053.950462:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879053.950494:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879053/real 1788879053] req@ffffa0618869b800 x1875771511575936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879053.950522:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511575936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879053.950534:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879053.950539:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879053.950544:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511575936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879053.950554:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879053.950586:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879053.950600:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879053/real 1788879053] req@ffffa06188699c00 x1875771511576064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879053.950615:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511576064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879053.950635:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879053.950639:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879053.950643:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879053.950652:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879053.950673:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879053.950685:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879053/real 1788879053] req@ffffa06188698380 x1875771511576192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879053.950699:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511576192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879053.950710:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879053.950714:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879053.950718:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879053.950726:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879053.950763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879053.950810:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879053/real 1788879053] req@ffffa0618869b100 x1875771511576320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879053.950833:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511576320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879053.950849:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879053.950855:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879053.950862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879054.589525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879054.589633:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879054.845511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879055.613511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879055.613739:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879055.869500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879056.637572:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879056.637726:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879056.893529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879057.661534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879057.661701:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879057.917559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879058.685520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879058.685671:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879058.941500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879059.069588:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879059.069603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879059.069610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879059.069627:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4045 00000100:00080000:1.0:1788879059.069646:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879059.069660:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879059.069721:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511576448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879059.069779:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879059.069787:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879059.069792:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879059.069801:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4045 00000100:00080000:1.0:1788879059.069810:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879059.069818:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879059.069836:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879059.069851:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511576448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879059.069862:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511576576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879059.069881:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879059.069887:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879059.069891:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879059.069897:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4045 00000100:00080000:1.0:1788879059.069903:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879059.069910:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879059.069918:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879059.069935:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511576704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879059.069952:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879059.069953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879059.069959:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879059.069962:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879059.069969:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4045 00000100:00080000:1.0:1788879059.069974:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879059.069981:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879059.069990:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879059.070008:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511576832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879059.070009:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879059/real 1788879059] req@ffffa0618869b100 x1875771511576448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879059.070028:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879059.070034:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879059.070037:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879059.070042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511576448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879059.070044:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4045 00000100:00080000:1.0:1788879059.070050:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879059.070058:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879059.070063:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788879059.070066:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879059.070070:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879059.070077:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511576448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879059.070084:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511576960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879059.070118:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879059.070163:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879059.070189:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879059/real 1788879059] req@ffffa06188698380 x1875771511576576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879059.070205:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511576576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879059.070217:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879059.070222:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879059.070226:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879059.070234:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879059.070259:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879059.070271:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879059/real 1788879059] req@ffffa06188699c00 x1875771511576704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879059.070285:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511576704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879059.070295:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879059.070298:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879059.070302:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879059.070308:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879059.070328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879059.070339:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879059/real 1788879059] req@ffffa0618869b800 x1875771511576832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879059.070352:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511576832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879059.070362:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879059.070365:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879059.070370:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879059.070377:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879059.070432:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879059.070446:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879059/real 1788879059] req@ffffa0618869aa00 x1875771511576960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879059.070461:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511576960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879059.070471:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879059.070475:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879059.070478:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511576960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879059.709540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879059.709804:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879059.965538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879060.733555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879060.733711:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879060.989516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879061.757530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879061.757645:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879062.013498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879062.781584:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879062.781730:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879063.037505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879063.805454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879063.805511:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879064.061479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879064.189557:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879064.189573:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879064.189579:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879064.189595:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4050 00000100:00080000:3.0:1788879064.189624:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879064.189635:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879064.189692:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511577088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879064.189763:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879064.189771:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879064.189774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879064.189781:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4050 00000100:00080000:3.0:1788879064.189788:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879064.189794:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879064.189808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511577088:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879064.189809:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879064.189836:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511577216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879064.189850:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879064.189857:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879064.189860:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879064.189866:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4050 00000100:00080000:3.0:1788879064.189870:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879064.189874:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879064.189880:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879064.189893:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511577344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879064.189903:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879064.189908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879064.189918:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879064.189921:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879064.189928:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4050 00000100:00080000:3.0:1788879064.189933:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879064.189939:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879064.189945:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879064.189959:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511577472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879064.189969:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879064.189974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879064/real 1788879064] req@ffffa06188872680 x1875771511577088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879064.189975:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879064.189980:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879064.189988:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4050 00000100:00080000:3.0:1788879064.189994:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879064.190002:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879064.190013:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879064.190032:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511577600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879064.190051:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511577088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879064.190067:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879064.190073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879064.190079:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511577088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879064.190110:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879064.190139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879064.190153:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879064/real 1788879064] req@ffffa06188872300 x1875771511577216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879064.190166:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511577216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879064.190175:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879064.190179:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879064.190182:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879064.190188:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879064.190229:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879064.190241:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879064/real 1788879064] req@ffffa06188873b80 x1875771511577344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879064.190253:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511577344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879064.190269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879064.190272:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879064.190276:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879064.190281:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879064.190299:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879064.190308:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879064/real 1788879064] req@ffffa06188870700 x1875771511577472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879064.190320:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511577472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879064.190329:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879064.190332:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879064.190335:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879064.190341:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879064.190356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879064.190364:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879064/real 1788879064] req@ffffa06188872d80 x1875771511577600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879064.190375:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511577600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879064.190411:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879064.190415:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879064.190419:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879064.829536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879064.829627:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879065.085488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879065.853526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879065.853711:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879066.109501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879066.877506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879066.877716:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879067.133513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879067.901520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879067.901721:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879068.157505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879068.925529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879068.925723:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879069.181515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879069.309567:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879069.309578:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879069.309586:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879069.309602:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4056 00000100:00080000:1.0:1788879069.309619:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879069.309631:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879069.309719:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511577728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879069.309769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879069.309778:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879069.309782:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879069.309791:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4056 00000100:00080000:1.0:1788879069.309800:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879069.309810:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879069.309827:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511577728:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879069.309827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879069.309859:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511577856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879069.309876:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879069.309881:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879069.309884:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879069.309890:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4056 00000100:00080000:1.0:1788879069.309921:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879069.309926:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879069.309933:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879069.309939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879069.309957:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511577984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879069.309971:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879069.309977:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879069.309981:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879069.309988:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4056 00000100:00080000:1.0:1788879069.309993:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879069.309997:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879069.310005:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879069.310023:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879069/real 1788879069] req@ffffa06188699f80 x1875771511577728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879069.310025:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511578112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879069.310094:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879069.310099:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879069.310103:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879069.310111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511577728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879069.310115:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4056 00000100:00080000:1.0:1788879069.310127:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879069.310139:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879069.310146:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788879069.310147:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879069.310163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879069.310172:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511578240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879069.310175:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511577728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879069.310226:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879069.310259:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879069.310275:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879069/real 1788879069] req@ffffa0618869a680 x1875771511577856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879069.310291:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511577856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879069.310302:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879069.310306:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879069.310312:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879069.310320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879069.310341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879069.310352:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879069/real 1788879069] req@ffffa0618869bb80 x1875771511577984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879069.310366:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511577984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879069.310376:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879069.310379:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879069.310413:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511577984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879069.310421:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879069.310446:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879069.310460:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879069/real 1788879069] req@ffffa0618869ad80 x1875771511578112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879069.310476:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511578112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879069.310486:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879069.310490:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879069.310494:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879069.310501:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879069.310519:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879069.310532:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879069/real 1788879069] req@ffffa06188698000 x1875771511578240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879069.310554:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511578240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879069.310571:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879069.310577:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879069.310600:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879069.949532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879069.949719:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879070.205488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879070.973539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879070.973672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879071.229477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879071.997557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879071.997755:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879072.253580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879073.021582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879073.021746:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879073.277514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879074.045552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879074.045622:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879074.301502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879074.429551:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879074.429564:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879074.429572:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879074.429609:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4061 00000100:00080000:1.0:1788879074.429654:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879074.429671:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879074.429732:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511578368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879074.429773:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879074.429778:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879074.429781:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879074.429787:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4061 00000100:00080000:1.0:1788879074.429793:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879074.429800:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879074.429814:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879074.429841:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511578496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879074.429846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511578368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879074.429884:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879074.429889:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879074.429891:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879074.429897:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4061 00000100:00080000:1.0:1788879074.429901:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879074.429909:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879074.429915:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879074.429937:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511578624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879074.429962:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879074.429966:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879074.429969:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879074.429979:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4061 00000800:00000100:0.0:1788879074.429984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879074.429988:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879074.429998:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879074.430008:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879074.430032:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511578752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879074.430045:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879074.430047:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879074/real 1788879074] req@ffffa06188698000 x1875771511578368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879074.430053:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879074.430058:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879074.430086:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4061 00000100:00080000:1.0:1788879074.430098:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879074.430105:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879074.430112:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511578368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879074.430113:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879074.430144:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879074.430153:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879074.430155:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511578880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879074.430167:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511578368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879074.430223:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879074.430256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879074.430270:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879074/real 1788879074] req@ffffa0618869ad80 x1875771511578496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879074.430283:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511578496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879074.430293:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879074.430297:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879074.430300:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879074.430307:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879074.430324:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879074.430333:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879074/real 1788879074] req@ffffa0618869bb80 x1875771511578624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879074.430345:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511578624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879074.430353:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879074.430356:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879074.430359:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879074.430365:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879074.430412:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879074.430425:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879074/real 1788879074] req@ffffa0618869a680 x1875771511578752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879074.430445:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511578752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879074.430455:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879074.430458:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879074.430461:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879074.430467:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879074.430485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879074.430496:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879074/real 1788879074] req@ffffa06188699f80 x1875771511578880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879074.430515:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511578880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879074.430528:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879074.430533:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879074.430539:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511578880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879075.069503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879075.069686:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879075.325510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879076.093532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879076.093712:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879076.349522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879077.117559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879077.117722:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879077.373466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879078.141517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879078.141729:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879078.397506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879079.165509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879079.165654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879079.421524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879079.549662:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879079.549682:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879079.549688:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879079.549701:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4066 00000100:00080000:3.0:1788879079.549734:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879079.549744:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879079.549801:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511579008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879079.549843:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879079.549847:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879079.549850:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879079.549855:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4066 00000100:00080000:3.0:1788879079.549860:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879079.549864:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879079.549878:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879079.549898:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511579136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879079.549920:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511579008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879079.549923:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879079.549949:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879079.549953:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879079.549959:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4066 00000100:00080000:3.0:1788879079.549965:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879079.549969:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879079.549975:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879079.549989:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511579264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879079.550000:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879079.550006:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879079.550008:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879079.550014:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4066 00000100:00080000:3.0:1788879079.550019:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879079.550038:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879079.550044:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879079.550057:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511579392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879079.550065:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879079.550069:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879079.550077:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879079.550086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879079.550115:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4066 00000100:00080000:3.0:1788879079.550122:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879079.550129:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879079.550139:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879079.550161:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511579520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879079.550186:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879079/real 1788879079] req@ffffa06188873800 x1875771511579008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879079.550236:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511579008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879079.550260:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879079.550266:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879079.550272:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511579008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879079.550309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879079.550349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879079.550367:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879079/real 1788879079] req@ffffa06188871880 x1875771511579136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879079.550419:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511579136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879079.550432:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879079.550436:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879079.550451:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879079.550460:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879079.550486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879079.550501:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879079/real 1788879079] req@ffffa06188870380 x1875771511579264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879079.550527:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511579264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879079.550546:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879079.550550:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879079.550554:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879079.550562:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879079.550581:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879079.550625:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879079/real 1788879079] req@ffffa06188870a80 x1875771511579392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879079.550651:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511579392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879079.550662:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879079.550666:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879079.550670:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879079.550678:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879079.550715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879079.550729:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879079/real 1788879079] req@ffffa06188871c00 x1875771511579520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879079.550742:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511579520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879079.550752:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879079.550756:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879079.550771:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879080.189602:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879080.189720:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879080.445519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879081.213513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879081.213656:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879081.469589:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879082.237563:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879082.237648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879082.493512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879083.261478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879083.261552:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879083.517484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879084.285463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879084.285594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879084.541441:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879084.669545:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879084.669589:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879084.669597:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879084.669619:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4071 00000100:00080000:3.0:1788879084.669642:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879084.669658:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879084.669722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511579648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879084.669763:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879084.669769:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879084.669772:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879084.669778:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4071 00000100:00080000:3.0:1788879084.669785:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879084.669792:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879084.669806:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879084.669819:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511579648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879084.669830:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511579776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879084.669857:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879084.669863:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879084.669868:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879084.669875:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4071 00000100:00080000:3.0:1788879084.669883:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879084.669892:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879084.669904:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879084.669933:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511579904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879084.669953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879084.669957:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879084.669964:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879084.669967:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879084.669976:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4071 00000100:00080000:3.0:1788879084.669984:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879084.669990:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879084.669999:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879084.670017:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879084/real 1788879084] req@ffffa06188871c00 x1875771511579648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879084.670025:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511580032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879084.670060:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879084.670063:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511579648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879084.670076:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879084.670086:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879084.670091:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879084.670101:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4071 00000100:00080000:0.0:1788879084.670105:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879084.670116:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511579648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879084.670119:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879084.670135:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879084.670144:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879084.670166:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511580160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879084.670172:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879084.670241:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879084.670278:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879084/real 1788879084] req@ffffa06188870a80 x1875771511579776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879084.670297:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511579776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879084.670309:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879084.670313:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879084.670319:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879084.670328:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879084.670361:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879084.670374:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879084/real 1788879084] req@ffffa06188870380 x1875771511579904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879084.670434:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511579904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879084.670445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879084.670449:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879084.670453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511579904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879084.670462:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879084.670487:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879084.670500:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879084/real 1788879084] req@ffffa06188871880 x1875771511580032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879084.670515:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511580032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879084.670525:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879084.670529:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879084.670532:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879084.670596:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879084.670625:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879084.670640:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879084/real 1788879084] req@ffffa06188873800 x1875771511580160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879084.670655:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511580160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879084.670666:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879084.670670:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879084.670674:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879085.309482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879085.309701:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879085.565487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879086.333593:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879086.333667:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879086.589601:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879087.357497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879087.357689:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879087.613496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879088.381526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879088.381635:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879088.637557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879089.405506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879089.405715:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879089.661566:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879089.789545:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879089.789557:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879089.789578:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879089.789590:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4076 00000100:00080000:1.0:1788879089.789603:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879089.789611:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879089.789658:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511580288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879089.789685:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879089.789688:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879089.789690:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879089.789694:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4076 00000100:00080000:1.0:1788879089.789696:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879089.789699:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879089.789709:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879089.789720:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511580416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879089.789725:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511580288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879089.789733:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879089.789737:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879089.789738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879089.789742:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4076 00000100:00080000:1.0:1788879089.789744:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879089.789747:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879089.789751:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879089.789760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511580544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879089.789773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879089.789776:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879089.789777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879089.789780:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4076 00000100:00080000:1.0:1788879089.789790:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879089.789792:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879089.789796:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879089.789803:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879089.789807:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511580672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879089.789815:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879089.789818:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879089.789820:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879089.789823:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4076 00000100:00080000:1.0:1788879089.789826:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879089.789829:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879089.789832:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879089.789840:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511580800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879089.789850:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879089/real 1788879089] req@ffffa0618869aa00 x1875771511580288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879089.789870:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511580288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879089.789881:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879089.789884:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879089.789888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511580288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879089.789911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879089.789928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879089.789938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879089/real 1788879089] req@ffffa0618869b800 x1875771511580416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879089.789947:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511580416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879089.789953:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879089.789956:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879089.789958:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879089.789962:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879089.789974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879089.789980:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879089/real 1788879089] req@ffffa06188699c00 x1875771511580544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879089.789988:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511580544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879089.789994:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879089.789996:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879089.789998:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879089.790001:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879089.790013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879089.790019:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879089/real 1788879089] req@ffffa06188698380 x1875771511580672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879089.790027:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511580672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879089.790034:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879089.790036:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879089.790038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879089.790042:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879089.790052:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879089.790058:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879089/real 1788879089] req@ffffa0618869b100 x1875771511580800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879089.790065:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511580800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879089.790072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879089.790073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879089.790075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511580800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879090.429542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879090.429649:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879090.685451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879091.453536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879091.453661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879091.709496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879092.477580:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879092.477724:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879092.733529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879093.501567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879093.501711:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879093.757627:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879094.525474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879094.525569:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879094.781547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879094.909611:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879094.909627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879094.909633:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879094.909647:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4081 00000100:00080000:1.0:1788879094.909667:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879094.909681:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879094.909775:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511580928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879094.909815:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879094.909820:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879094.909823:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879094.909829:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4081 00000100:00080000:1.0:1788879094.909836:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879094.909842:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879094.909856:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879094.909867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511580928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879094.909882:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511581056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879094.909900:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879094.909905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879094.909908:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879094.909913:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4081 00000100:00080000:1.0:1788879094.909920:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879094.909924:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879094.909949:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879094.909962:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511581184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879094.909974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879094.909990:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879094.909995:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879094.909997:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879094.910005:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4081 00000100:00080000:1.0:1788879094.910009:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879094.910015:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879094.910025:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879094.910035:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879094/real 1788879094] req@ffffa0618869b100 x1875771511580928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879094.910038:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511581312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879094.910112:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879094.910115:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879094.910118:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879094.910125:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511580928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879094.910131:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4081 00000100:00080000:1.0:1788879094.910143:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879094.910160:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879094.910173:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879094.910174:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879094.910190:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879094.910196:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511581440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879094.910200:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511580928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879094.910247:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879094.910289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879094.910306:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879094/real 1788879094] req@ffffa06188698380 x1875771511581056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879094.910318:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511581056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879094.910326:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879094.910329:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879094.910331:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879094.910336:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879094.910349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879094.910356:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879094/real 1788879094] req@ffffa06188699c00 x1875771511581184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879094.910365:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511581184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879094.910391:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879094.910431:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879094.910451:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879094.910457:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879094.910478:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879094.910504:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879094/real 1788879094] req@ffffa0618869b800 x1875771511581312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879094.910520:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511581312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879094.910541:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879094.910563:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879094.910570:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879094.910582:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879094.910627:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879094.910642:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879094/real 1788879094] req@ffffa0618869aa00 x1875771511581440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879094.910656:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511581440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879094.910667:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879094.910670:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879094.910674:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581440:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788879095.549674:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879095.549704:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788879095.805502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879096.573467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879096.573645:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879096.829559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879097.597462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879097.597562:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879097.853510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879098.621570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879098.621728:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879098.877523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879099.645528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879099.645656:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879099.901563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879100.029624:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879100.029640:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879100.029649:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879100.029666:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4086 00000100:00080000:3.0:1788879100.029684:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879100.029696:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879100.029756:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511581568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879100.029801:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879100.029807:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879100.029809:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879100.029815:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4086 00000100:00080000:3.0:1788879100.029819:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879100.029824:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879100.029838:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879100.029856:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511581696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879100.029871:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879100.029875:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879100.029878:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879100.029883:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4086 00000100:00080000:3.0:1788879100.029887:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879100.029896:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511581568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879100.029897:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879100.029912:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879100.029927:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511581824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879100.029949:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879100.029955:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879100.029958:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879100.029963:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4086 00000100:00080000:3.0:1788879100.029967:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879100.029972:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879100.029978:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879100.029992:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511581952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879100.030005:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879100.030013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879100.030015:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879100.030021:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879100.030027:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4086 00000100:00080000:3.0:1788879100.030032:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879100.030037:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879100.030044:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879100.030059:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511582080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879100.030079:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879100/real 1788879100] req@ffffa06188872d80 x1875771511581568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879100.030110:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511581568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879100.030126:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879100.030132:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879100.030138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511581568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879100.030147:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879100.030176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879100.030191:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879100/real 1788879100] req@ffffa06188870700 x1875771511581696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879100.030206:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511581696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879100.030217:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879100.030221:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879100.030225:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879100.030258:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879100.030279:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879100.030290:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879100/real 1788879100] req@ffffa06188873b80 x1875771511581824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879100.030305:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511581824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879100.030324:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879100.030328:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879100.030332:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879100.030339:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879100.030358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879100.030369:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879100/real 1788879100] req@ffffa06188872300 x1875771511581952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879100.030422:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511581952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879100.030434:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879100.030438:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879100.030442:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511581952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879100.030474:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879100.030501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879100.030516:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879100/real 1788879100] req@ffffa06188872680 x1875771511582080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879100.030531:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511582080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879100.030542:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879100.030545:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879100.030550:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879100.669568:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879100.669712:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879100.925595:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879101.693484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879101.693562:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879101.949565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879102.717550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879102.717660:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879102.973526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879103.741477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879103.741556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879103.997516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879104.765515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879104.765627:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879105.021570:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879105.149503:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879105.149510:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879105.149514:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879105.149524:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4091 00000100:00080000:3.0:1788879105.149535:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879105.149542:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879105.149577:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511582208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879105.149600:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879105.149603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879105.149605:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879105.149608:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4091 00000100:00080000:3.0:1788879105.149610:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879105.149613:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879105.149622:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879105.149632:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511582336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879105.149636:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511582208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879105.149645:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879105.149648:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879105.149649:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879105.149653:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4091 00000100:00080000:3.0:1788879105.149655:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879105.149657:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879105.149661:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879105.149669:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511582464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879105.149676:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879105.149679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879105.149680:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879105.149683:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4091 00000100:00080000:3.0:1788879105.149686:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879105.149688:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879105.149692:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879105.149701:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511582592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879105.149706:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879105.149709:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879105.149710:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879105.149712:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4091 00000100:00080000:3.0:1788879105.149715:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879105.149718:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879105.149733:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879105.149746:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511582720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879105.149753:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879105.149794:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879105.149815:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879105.149830:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879105.789551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879105.789665:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879106.045549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879106.813531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879106.813622:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879107.069518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879107.837509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879107.837618:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879108.093545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788879108.221670:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788879108.221726:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788879108.221744:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879108.221792:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879108.221805:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879108.221818:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879108.221832:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879108.221864:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879105/real 1788879108] req@ffffa06188872680 x1875771511582720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879108.221909:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511582720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879108.221927:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879108.221935:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879108.221942:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879108.221954:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879105/real 1788879108] req@ffffa06188873100 x1875771511582336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879108.221973:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511582336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879108.221986:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879108.221991:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879108.221997:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879108.222008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879105/real 1788879108] req@ffffa06188871500 x1875771511582464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879108.222024:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511582464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879108.222036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879108.222041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879108.222046:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879108.222056:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879105/real 1788879108] req@ffffa06188873480 x1875771511582592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879108.222075:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511582592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879108.222087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879108.222092:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879108.222097:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879108.222108:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879105/real 1788879108] req@ffffa06188872a00 x1875771511582208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879108.222125:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511582208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879108.222137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879108.222142:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879108.222147:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511582208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879108.861553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879108.861661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879109.117633:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879109.885552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879109.885674:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879110.141509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879110.269681:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879110.269696:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879110.269703:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879110.269718:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4096 00000100:00080000:1.0:1788879110.269735:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879110.269748:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879110.269798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511582848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879110.269833:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879110.269839:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879110.269841:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879110.269847:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4096 00000100:00080000:1.0:1788879110.269851:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879110.269857:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879110.269871:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879110.269890:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511582976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879110.269903:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511582848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879110.269911:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879110.269919:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879110.269922:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879110.269928:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4096 00000100:00080000:1.0:1788879110.269932:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879110.269937:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879110.269945:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879110.269969:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511583104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879110.269988:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879110.269995:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879110.269999:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879110.270005:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4096 00000100:00080000:1.0:1788879110.270013:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879110.270020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879110.270029:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879110.270042:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879110.270094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879110/real 1788879110] req@ffffa06188699f80 x1875771511582848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879110.270096:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511583232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879110.270174:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879110.270181:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879110.270184:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879110.270193:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511582848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879110.270202:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4096 00000100:00080000:0.0:1788879110.270228:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879110.270230:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879110.270251:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879110.270253:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879110.270273:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511582848:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879110.270274:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879110.270315:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511583360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879110.270331:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879110.270491:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879110.270522:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879110/real 1788879110] req@ffffa0618869a680 x1875771511582976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879110.270549:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511582976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879110.270567:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879110.270574:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879110.270582:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511582976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879110.270595:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879110.270640:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879110.270665:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879110/real 1788879110] req@ffffa0618869bb80 x1875771511583104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879110.270691:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511583104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879110.270708:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879110.270714:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879110.270721:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879110.270734:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879110.270772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879110.270794:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879110/real 1788879110] req@ffffa0618869ad80 x1875771511583232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879110.270818:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511583232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879110.270834:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879110.270840:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879110.270847:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879110.270880:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879110.270918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879110.270938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879110/real 1788879110] req@ffffa06188698000 x1875771511583360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879110.270961:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511583360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879110.270990:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879110.270997:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879110.271022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879110.909532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879110.909660:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879111.165505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879111.933530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879111.933632:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879112.189471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879112.957530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879112.957566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879113.213496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879113.981500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879113.981652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879114.237553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879115.005563:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879115.005697:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879115.261513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879115.389603:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879115.389618:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879115.389625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879115.389641:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4102 00000100:00080000:3.0:1788879115.389656:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879115.389668:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879115.389749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511583488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879115.389805:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879115.389812:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879115.389816:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879115.389825:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4102 00000100:00080000:3.0:1788879115.389842:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879115.389854:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879115.389872:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879115.389902:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511583616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879115.389907:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511583488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879115.389940:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879115.389949:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879115.389953:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879115.389962:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4102 00000100:00080000:3.0:1788879115.389970:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879115.389980:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879115.389990:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879115.390013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879115.390021:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511583744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879115.390074:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879115.390081:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879115/real 1788879115] req@ffffa06188872300 x1875771511583488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879115.390094:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879115.390101:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879115.390129:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4102 00000100:00100000:0.0:1788879115.390135:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511583488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879115.390141:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879115.390160:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879115.390161:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879115.390178:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788879115.390182:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879115.390185:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511583488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879115.390219:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511583872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879115.390229:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583616:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788879115.390268:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879115.390277:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788879115.390283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879115.390283:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879115.390301:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4102 00000100:00100000:0.0:1788879115.390311:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879115/real 1788879115] req@ffffa06188873b80 x1875771511583616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879115.390313:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879115.390334:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879115.390370:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511583616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879115.390433:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879115.390447:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879115.390458:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879115.390466:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879115.390485:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879115.390486:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511584000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879115.390520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879115.390539:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879115/real 1788879115] req@ffffa06188870700 x1875771511583744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879115.390556:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511583744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879115.390566:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879115.390571:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879115.390575:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879115.390608:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879115.390626:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879115.390636:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879115/real 1788879115] req@ffffa06188872d80 x1875771511583872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879115.390647:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511583872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879115.390674:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879115.390678:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879115.390682:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511583872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879115.390689:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879115.390708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879115.390719:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879115/real 1788879115] req@ffffa06188873800 x1875771511584000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879115.390733:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511584000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879115.390743:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879115.390746:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879115.390750:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879116.029477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879116.029545:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879116.285489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879117.053498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879117.053612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879117.309508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879118.077480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879118.077564:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879118.333490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879119.101508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879119.101614:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879119.357533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879120.125453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879120.125593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879120.381477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879120.509593:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879120.509605:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879120.509612:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879120.509628:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4107 00000100:00080000:2.0:1788879120.509643:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879120.509674:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879120.509746:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511584128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879120.509804:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879120.509809:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879120.509811:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879120.509816:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4107 00000100:00080000:2.0:1788879120.509820:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879120.509825:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879120.509836:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879120.509853:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511584256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879120.509865:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879120.509868:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879120.509870:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879120.509875:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4107 00000100:00080000:2.0:1788879120.509880:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879120.509888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511584128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879120.509893:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879120.509905:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879120.509929:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511584384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879120.509947:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879120.509953:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879120.509958:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879120.509964:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4107 00000100:00080000:2.0:1788879120.509972:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879120.509977:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879120.509984:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879120.509992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879120.510005:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511584512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879120.510019:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879120.510022:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879120.510024:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879120.510029:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4107 00000100:00080000:2.0:1788879120.510034:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879120.510039:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879120.510061:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879120.510063:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879120/real 1788879120] req@ffffa061888ac380 x1875771511584128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879120.510074:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511584640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879120.510086:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511584128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879120.510098:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879120.510102:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879120.510105:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511584128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879120.510112:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879120.510132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879120.510141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879120/real 1788879120] req@ffffa061888ae680 x1875771511584256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879120.510150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511584256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879120.510162:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879120.510165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879120.510168:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879120.510189:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879120.510201:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879120.510209:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879120/real 1788879120] req@ffffa061888adc00 x1875771511584384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879120.510217:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511584384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879120.510224:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879120.510226:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879120.510228:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879120.510233:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879120.510245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879120.510251:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879120/real 1788879120] req@ffffa061888afb80 x1875771511584512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879120.510264:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511584512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879120.510274:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879120.510278:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879120.510283:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879120.510291:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879120.510331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879120.510363:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879120/real 1788879120] req@ffffa061888aea00 x1875771511584640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879120.510377:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511584640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879120.510386:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879120.510389:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879120.510393:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879121.149541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879121.149667:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879121.405557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879122.173455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879122.173525:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879122.429502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879123.197467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879123.197534:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879123.453446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879124.221450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879124.221497:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879124.477460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879125.245446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879125.245519:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879125.501450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879125.629487:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879125.629496:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879125.629500:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879125.629508:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4112 00000100:00080000:1.0:1788879125.629519:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879125.629526:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879125.629564:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511584768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879125.629587:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879125.629590:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879125.629592:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879125.629595:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4112 00000100:00080000:1.0:1788879125.629598:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879125.629601:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879125.629610:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879125.629622:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511584896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879125.629623:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511584768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879125.629638:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879125.629641:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879125.629642:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879125.629644:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4112 00000100:00080000:1.0:1788879125.629646:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879125.629648:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879125.629652:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879125.629659:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511585024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879125.629665:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879125.629668:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879125.629669:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879125.629672:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4112 00000100:00080000:1.0:1788879125.629687:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879125.629691:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879125.629695:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879125.629698:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879125.629707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511585152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879125.629715:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879125.629717:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879125.629718:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879125.629721:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4112 00000100:00080000:1.0:1788879125.629724:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879125.629727:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879125.629731:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879125.629735:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879125/real 1788879125] req@ffffa06188698000 x1875771511584768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879125.629740:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511585280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879125.629761:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511584768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879125.629771:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879125.629775:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879125.629778:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511584768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879125.629799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879125.629815:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879125.629824:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879125/real 1788879125] req@ffffa0618869ad80 x1875771511584896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879125.629833:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511584896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879125.629839:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879125.629841:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879125.629844:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511584896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879125.629848:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879125.629858:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879125.629864:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879125/real 1788879125] req@ffffa0618869bb80 x1875771511585024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879125.629871:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511585024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879125.629876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879125.629878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879125.629880:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879125.629883:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879125.629895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879125.629900:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879125/real 1788879125] req@ffffa0618869a680 x1875771511585152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879125.629908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511585152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879125.629914:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879125.629916:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879125.629918:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879125.629921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879125.629930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879125.629935:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879125/real 1788879125] req@ffffa06188699f80 x1875771511585280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879125.629942:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511585280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879125.629948:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879125.629950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879125.629951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879126.269470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879126.269547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879126.525470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879127.293490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879127.293577:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879127.549498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879128.317492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879128.317558:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879128.573467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879129.341477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879129.341603:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879129.597510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879130.365470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879130.365547:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879130.621461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879130.749477:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879130.749485:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879130.749489:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879130.749500:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4117 00000100:00080000:1.0:1788879130.749512:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879130.749519:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879130.749569:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511585408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879130.749593:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879130.749597:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879130.749598:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879130.749602:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4117 00000100:00080000:1.0:1788879130.749604:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879130.749607:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879130.749616:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879130.749625:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511585536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879130.749634:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879130.749637:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879130.749639:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879130.749643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511585408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879130.749644:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4117 00000100:00080000:1.0:1788879130.749649:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879130.749655:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879130.749659:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879130.749667:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511585664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879130.749673:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879130.749676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879130.749677:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879130.749680:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4117 00000100:00080000:1.0:1788879130.749684:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879130.749687:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879130.749690:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879130.749698:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511585792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879130.749703:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879130.749705:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879130.749707:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879130.749709:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4117 00000100:00080000:1.0:1788879130.749720:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879130.749734:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879130.749738:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879130.749743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879130.749750:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511585920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879130.749790:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879130/real 1788879130] req@ffffa0618869aa00 x1875771511585408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879130.749812:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511585408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879130.749825:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879130.749830:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879130.749834:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511585408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879130.749860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879130.749889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879130.749903:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879130/real 1788879130] req@ffffa0618869b800 x1875771511585536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879130.749918:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511585536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879130.749928:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879130.749932:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879130.749936:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879130.749944:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879130.749966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879130.749978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879130/real 1788879130] req@ffffa06188699c00 x1875771511585664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879130.749992:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511585664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879130.750002:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879130.750006:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879130.750010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879130.750016:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879130.750034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879130.750044:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879130/real 1788879130] req@ffffa06188698380 x1875771511585792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879130.750058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511585792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879130.750068:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879130.750071:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879130.750075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879130.750081:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879130.750098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879130.750119:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879130/real 1788879130] req@ffffa0618869b100 x1875771511585920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879130.750132:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511585920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879130.750142:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879130.750145:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879130.750149:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511585920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879131.389460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879131.389545:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879131.645439:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879132.413484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879132.413630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879132.669529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879133.437508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879133.437605:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879133.693545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879134.461494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879134.461628:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879134.717498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879135.485537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879135.485728:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879135.741521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879135.869513:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879135.869525:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879135.869530:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879135.869542:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4122 00000100:00080000:3.0:1788879135.869567:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879135.869576:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879135.869643:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511586048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879135.869669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879135.869677:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879135.869680:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879135.869685:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4122 00000100:00080000:3.0:1788879135.869708:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879135.869714:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879135.869724:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511586048:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879135.869725:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879135.869742:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511586176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879135.869766:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879135.869770:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879135.869772:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879135.869776:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4122 00000100:00080000:3.0:1788879135.869780:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879135.869784:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879135.869789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879135.869801:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511586304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879135.869811:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879135.869813:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879135.869815:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879135.869817:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879135.869821:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4122 00000100:00080000:3.0:1788879135.869825:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879135.869839:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879135.869846:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879135.869857:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511586432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879135.869863:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879135/real 1788879135] req@ffffa06188871880 x1875771511586048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879135.869868:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879135.869873:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879135.869875:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879135.869880:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4122 00000100:00080000:3.0:1788879135.869883:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879135.869886:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511586048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879135.869887:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879135.869902:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879135.869914:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788879135.869915:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511586560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879135.869919:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879135.869924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511586048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879135.869954:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879135.869985:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879135.870011:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879135/real 1788879135] req@ffffa06188870380 x1875771511586176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879135.870022:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511586176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879135.870029:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879135.870032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879135.870034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879135.870039:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879135.870055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879135.870074:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879135/real 1788879135] req@ffffa06188870a80 x1875771511586304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879135.870082:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511586304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879135.870089:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879135.870091:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879135.870093:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879135.870098:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879135.870112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879135.870119:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879135/real 1788879135] req@ffffa06188871c00 x1875771511586432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879135.870128:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511586432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879135.870134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879135.870137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879135.870139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879135.870143:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879135.870166:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879135.870175:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879135/real 1788879135] req@ffffa06188873800 x1875771511586560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879135.870183:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511586560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879135.870190:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879135.870192:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879135.870222:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879136.509506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879136.509640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879136.765503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879137.533520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879137.533583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879137.789464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879138.557549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879138.557630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879138.813520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879139.581466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879139.581543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879139.837513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879140.605525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879140.605614:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879140.861563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879140.989563:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879140.989591:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879140.989599:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879140.989616:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4127 00000100:00080000:3.0:1788879140.989648:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879140.989658:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879140.989741:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511586688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879140.989783:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879140.989788:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879140.989791:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879140.989798:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4127 00000100:00080000:3.0:1788879140.989806:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879140.989831:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879140.989845:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879140.989859:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511586688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879140.989866:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511586816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879140.989908:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879140.989914:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879140.989917:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879140.989924:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4127 00000100:00080000:3.0:1788879140.989931:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879140.989938:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879140.989957:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879140.989973:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511586944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879140.989986:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879140.989992:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788879140.990000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879140.990002:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879140.990010:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4127 00000100:00080000:3.0:1788879140.990025:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879140.990032:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879140.990053:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879140.990074:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511587072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879140.990078:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879140/real 1788879140] req@ffffa06188872d80 x1875771511586688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879140.990226:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511586688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879140.990230:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879140.990274:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879140.990275:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879140.990290:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879140.990291:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879140.990307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511586688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879140.990317:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4127 00000100:00080000:3.0:1788879140.990331:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879140.990337:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879140.990346:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879140.990349:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879140.990375:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511587200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879140.990393:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879140.990464:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879140/real 1788879140] req@ffffa06188870700 x1875771511586816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879140.990492:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511586816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879140.990503:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879140.990507:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879140.990511:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879140.990519:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879140.990542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879140.990553:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879140/real 1788879140] req@ffffa06188873b80 x1875771511586944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879140.990565:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511586944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879140.990579:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879140.990583:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879140.990586:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511586944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879140.990592:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879140.990609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879140.990629:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879140/real 1788879140] req@ffffa06188872300 x1875771511587072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879140.990640:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511587072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879140.990648:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879140.990651:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879140.990654:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879140.990671:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879140.990699:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879140.990709:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879140/real 1788879140] req@ffffa06188872a00 x1875771511587200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879140.990721:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511587200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879140.990729:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879140.990732:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879140.990736:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879141.629485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879141.629554:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879141.885560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879142.653548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879142.653640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879142.909479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879143.677478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879143.677610:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879143.933508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879144.701528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879144.701670:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879144.957455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879145.725540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879145.725702:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879145.981482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788879146.109492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879146.109500:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788879146.109503:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879146.109512:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4132 00000100:00080000:0.0:1788879146.109522:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879146.109528:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879146.109563:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511587328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879146.109585:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879146.109588:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788879146.109589:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879146.109593:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4132 00000100:00080000:0.0:1788879146.109595:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879146.109598:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879146.109606:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879146.109618:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511587456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879146.109618:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511587328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788879146.109631:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879146.109634:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788879146.109635:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879146.109638:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4132 00000100:00080000:0.0:1788879146.109641:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879146.109644:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879146.109647:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879146.109655:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511587584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879146.109662:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879146.109664:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788879146.109666:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879146.109668:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4132 00000100:00080000:0.0:1788879146.109670:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879146.109672:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879146.109676:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879146.109684:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511587712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788879146.109690:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788879146.109695:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879146.109699:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879146.109700:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879146.109704:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4132 00000100:00080000:0.0:1788879146.109706:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879146.109709:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879146.109712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879146.109719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511587840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879146.109728:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879146/real 1788879146] req@ffffa06198afd180 x1875771511587328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879146.109746:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511587328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879146.109754:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879146.109758:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879146.109761:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511587328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879146.109779:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879146.109794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879146.109802:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879146/real 1788879146] req@ffffa06198afd500 x1875771511587456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879146.109810:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511587456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879146.109816:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879146.109818:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879146.109820:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879146.109823:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879146.109834:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879146.109840:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879146/real 1788879146] req@ffffa06198afca80 x1875771511587584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879146.109847:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511587584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879146.109852:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879146.109853:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879146.109855:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879146.109859:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879146.109869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879146.109874:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879146/real 1788879146] req@ffffa06198afc380 x1875771511587712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879146.109881:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511587712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879146.109891:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879146.109893:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879146.109895:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879146.109898:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879146.109907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879146.109913:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879146/real 1788879146] req@ffffa06198afe300 x1875771511587840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879146.109920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511587840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879146.109925:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879146.109926:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879146.109928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511587840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879146.749485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879146.749606:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879147.005464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879147.773467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879147.773568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879148.029495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879148.797467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879148.797516:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879149.053459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879149.821529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879149.821578:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879150.077463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879150.845473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879150.845550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879151.101477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879151.229511:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879151.229523:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879151.229528:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879151.229552:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4137 00000100:00080000:2.0:1788879151.229565:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879151.229574:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879151.229626:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511587968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879151.229660:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879151.229666:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879151.229668:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879151.229673:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4137 00000100:00080000:2.0:1788879151.229677:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879151.229682:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879151.229693:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879151.229706:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511588096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879151.229719:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879151.229723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879151.229727:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788879151.229731:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511587968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879151.229737:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4137 00000100:00080000:2.0:1788879151.229748:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879151.229753:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879151.229762:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879151.229782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511588224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879151.229797:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879151.229802:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879151.229804:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879151.229809:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4137 00000100:00080000:2.0:1788879151.229814:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879151.229819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879151.229826:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788879151.229834:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879151.229845:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511588352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879151.229856:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879151.229859:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879151.229861:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879151.229865:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4137 00000100:00080000:2.0:1788879151.229869:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879151.229873:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879151.229878:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879151.229886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879151/real 1788879151] req@ffffa061888ae300 x1875771511587968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879151.229891:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511588480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879151.229943:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511587968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879151.229961:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879151.229967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879151.229973:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511587968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879151.230015:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879151.230050:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879151.230069:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879151/real 1788879151] req@ffffa061888af100 x1875771511588096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879151.230089:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511588096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879151.230173:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879151.230182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879151.230189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879151.230211:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879151.230245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879151.230260:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879151/real 1788879151] req@ffffa061888ad500 x1875771511588224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879151.230273:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511588224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879151.230281:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879151.230284:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879151.230287:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879151.230294:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879151.230321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879151.230338:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879151/real 1788879151] req@ffffa061888ac000 x1875771511588352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879151.230357:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511588352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879151.230369:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879151.230374:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879151.230377:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879151.230409:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879151.230432:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879151.230443:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879151/real 1788879151] req@ffffa061888adf80 x1875771511588480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879151.230454:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511588480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879151.230462:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879151.230465:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879151.230467:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879151.869493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879151.869740:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879152.125490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879152.893521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879152.893613:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879153.149543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879153.917510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879153.917631:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879154.173514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879154.941555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879154.941640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879155.197517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879155.965507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879155.965697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879156.221502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879156.349509:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879156.349524:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879156.349530:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879156.349547:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4143 00000100:00080000:2.0:1788879156.349566:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879156.349577:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879156.349630:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511588608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879156.349676:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879156.349681:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879156.349684:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879156.349689:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4143 00000100:00080000:2.0:1788879156.349696:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879156.349702:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879156.349717:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879156.349735:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511588736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879156.349753:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879156.349755:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511588608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879156.349760:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879156.349768:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879156.349776:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4143 00000100:00080000:2.0:1788879156.349779:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879156.349783:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879156.349791:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879156.349823:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511588864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879156.349837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879156.349843:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879156.349845:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879156.349850:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4143 00000100:00080000:2.0:1788879156.349856:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879156.349864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879156.349870:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879156.349880:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879156.349897:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511588992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879156.349911:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879156.349916:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879156.349918:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879156.349928:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4143 00000100:00100000:0.0:1788879156.349930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879156/real 1788879156] req@ffffa061888af480 x1875771511588608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879156.349932:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879156.349944:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879156.349958:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879156.349978:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511588608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879156.349987:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511589120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879156.350022:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879156.350032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879156.350039:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511588608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879156.350104:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879156.350148:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879156.350166:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879156/real 1788879156] req@ffffa061888aca80 x1875771511588736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879156.350182:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511588736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879156.350193:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879156.350197:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879156.350202:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879156.350213:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879156.350234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879156.350246:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879156/real 1788879156] req@ffffa061888adf80 x1875771511588864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879156.350260:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511588864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879156.350270:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879156.350274:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879156.350277:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879156.350284:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879156.350303:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879156.350314:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879156/real 1788879156] req@ffffa061888ac000 x1875771511588992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879156.350328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511588992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879156.350337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879156.350341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879156.350344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511588992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879156.350351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879156.350368:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879156.350410:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879156/real 1788879156] req@ffffa061888ad500 x1875771511589120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879156.350428:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511589120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879156.350438:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879156.350442:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879156.350445:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879156.989583:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879156.989685:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879157.245498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879158.013525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879158.013637:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879158.269503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879159.037508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879159.037705:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879159.293584:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879160.061557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879160.061684:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879160.317462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879161.085548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879161.085622:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879161.341498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879161.469662:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879161.469678:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879161.469684:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879161.469700:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4148 00000100:00080000:1.0:1788879161.469718:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879161.469729:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879161.469795:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511589248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879161.469844:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879161.469849:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879161.469852:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879161.469858:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4148 00000100:00080000:1.0:1788879161.469862:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879161.469867:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879161.469881:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879161.469897:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511589376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879161.469923:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879161.469928:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879161.469931:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879161.469935:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4148 00000100:00080000:1.0:1788879161.469939:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879161.469943:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879161.469951:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879161.469965:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511589504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879161.469981:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511589248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879161.469985:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879161.470013:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879161.470017:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879161.470023:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4148 00000100:00080000:1.0:1788879161.470054:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879161.470060:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879161.470067:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879161.470083:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511589632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879161.470096:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879161.470101:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879161.470106:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879161.470112:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4148 00000100:00080000:1.0:1788879161.470130:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879161.470136:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879161.470145:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879161.470159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879161.470169:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511589760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879161.470233:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879161/real 1788879161] req@ffffa0618869b100 x1875771511589248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879161.470260:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511589248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879161.470277:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879161.470286:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879161.470294:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511589248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879161.470309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879161.470358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879161.470429:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879161/real 1788879161] req@ffffa06188698380 x1875771511589376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879161.470450:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511589376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879161.470463:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879161.470467:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879161.470472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879161.470508:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879161.470538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879161.470553:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879161/real 1788879161] req@ffffa06188699c00 x1875771511589504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879161.470567:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511589504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879161.470577:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879161.470581:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879161.470584:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879161.470592:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879161.470613:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879161.470624:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879161/real 1788879161] req@ffffa0618869b800 x1875771511589632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879161.470638:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511589632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879161.470648:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879161.470651:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879161.470656:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879161.470664:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879161.470682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879161.470693:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879161/real 1788879161] req@ffffa0618869aa00 x1875771511589760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879161.470706:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511589760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879161.470717:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879161.470720:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879161.470724:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511589760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879162.109528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879162.109628:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879162.365536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879163.133549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879163.133740:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879163.389495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879164.157518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879164.157681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879164.413514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879165.181430:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879165.181551:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879165.437448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879166.205494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879166.205588:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879166.461526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879166.589491:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879166.589503:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879166.589509:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879166.589524:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4153 00000100:00080000:1.0:1788879166.589538:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879166.589548:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879166.589608:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511589888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879166.589643:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879166.589647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879166.589650:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879166.589655:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4153 00000100:00080000:1.0:1788879166.589659:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879166.589665:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879166.589679:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879166.589697:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511589888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879166.589699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511590016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879166.589723:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879166.589727:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879166.589729:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879166.589734:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4153 00000100:00080000:1.0:1788879166.589739:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879166.589743:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879166.589749:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879166.589765:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511590144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879166.589776:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879166.589781:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879166.589784:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879166.589788:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4153 00000100:00080000:1.0:1788879166.589792:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879166.589800:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879166.589801:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879166.589812:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879166.589828:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511590272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879166.589841:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879166.589845:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879166.589848:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879166.589856:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4153 00000100:00100000:0.0:1788879166.589857:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879166/real 1788879166] req@ffffa06188699f80 x1875771511589888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879166.589865:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879166.589879:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879166.589889:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879166.589898:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511589888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879166.589906:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511590400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879166.589927:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879166.589938:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879166.589955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511589888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879166.590013:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879166.590064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879166.590088:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879166/real 1788879166] req@ffffa0618869a680 x1875771511590016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879166.590104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511590016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879166.590115:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879166.590119:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879166.590123:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879166.590130:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879166.590155:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879166.590167:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879166/real 1788879166] req@ffffa0618869bb80 x1875771511590144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879166.590179:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511590144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879166.590188:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879166.590192:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879166.590195:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879166.590202:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879166.590220:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879166.590230:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879166/real 1788879166] req@ffffa0618869ad80 x1875771511590272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879166.590260:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511590272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879166.590291:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879166.590297:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879166.590304:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879166.590315:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879166.590352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879166.590368:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879166/real 1788879166] req@ffffa06188698000 x1875771511590400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879166.590413:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511590400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879166.590424:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879166.590428:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879166.590432:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879167.229491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879167.229556:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879167.485508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879168.253496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879168.253565:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879168.509509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879169.277522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879169.277677:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879169.533504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879170.301551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879170.301591:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879170.557459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879171.325458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879171.325557:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879171.581452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879171.709496:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879171.709504:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879171.709508:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879171.709517:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4158 00000100:00080000:3.0:1788879171.709526:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879171.709533:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879171.709567:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511590528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879171.709591:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879171.709594:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879171.709595:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879171.709598:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4158 00000100:00080000:3.0:1788879171.709600:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879171.709603:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879171.709612:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879171.709622:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511590656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879171.709626:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511590528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879171.709632:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879171.709636:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879171.709637:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879171.709640:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4158 00000100:00080000:3.0:1788879171.709641:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879171.709644:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879171.709647:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879171.709656:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511590784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879171.709661:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879171.709664:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879171.709665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879171.709667:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4158 00000100:00080000:3.0:1788879171.709670:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879171.709672:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879171.709676:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879171.709683:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511590912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879171.709688:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879171.709691:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879171.709692:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879171.709694:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4158 00000100:00080000:3.0:1788879171.709696:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879171.709699:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879171.709701:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879171.709709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511591040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879171.709737:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879171.709759:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879171.709773:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879171.709791:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879172.349491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879172.349594:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879172.605482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879173.373545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879173.373643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879173.629478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879174.397526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879174.397635:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879174.653480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788879174.781632:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788879174.781677:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788879174.781695:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879174.781733:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879174.781751:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879174.781757:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879171/real 1788879174] req@ffffa06188873480 x1875771511590528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1788879174.781771:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879174.781788:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511590528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1788879174.781796:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00080000:0.0:1788879174.781810:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879174.781817:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879174.781822:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511590528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879174.781844:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879171/real 1788879174] req@ffffa06188871500 x1875771511590656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879174.781859:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511590656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879174.781867:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879174.781871:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879174.781875:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879174.781883:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879171/real 1788879174] req@ffffa06188873100 x1875771511590784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879174.781894:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511590784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879174.781902:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879174.781906:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879174.781919:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879174.781927:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879171/real 1788879174] req@ffffa06188872680 x1875771511590912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879174.781961:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511590912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879174.781969:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879174.781973:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879174.781976:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511590912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879174.781983:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879171/real 1788879174] req@ffffa06188872a00 x1875771511591040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879174.781996:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511591040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879174.782004:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879174.782007:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879174.782011:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879175.421513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879175.421663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879175.677455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879176.445495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879176.445635:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879176.701464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879176.829529:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879176.829538:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879176.829541:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879176.829550:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4163 00000100:00080000:1.0:1788879176.829559:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879176.829566:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879176.829597:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511591168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879176.829623:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879176.829626:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879176.829627:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879176.829630:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4163 00000100:00080000:1.0:1788879176.829633:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879176.829635:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879176.829643:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879176.829651:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511591296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879176.829659:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879176.829663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788879176.829665:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511591168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879176.829665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879176.829673:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4163 00000100:00080000:1.0:1788879176.829676:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879176.829678:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879176.829682:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879176.829690:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511591424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879176.829697:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879176.829699:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879176.829701:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879176.829703:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4163 00000100:00080000:1.0:1788879176.829719:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879176.829722:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879176.829725:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879176.829729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879176.829736:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511591552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879176.829743:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879176.829746:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879176.829747:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879176.829750:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4163 00000100:00080000:1.0:1788879176.829753:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879176.829756:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879176.829759:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879176.829763:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879176/real 1788879176] req@ffffa06188698000 x1875771511591168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879176.829768:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511591680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879176.829807:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511591168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879176.829818:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879176.829822:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879176.829837:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511591168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879176.829865:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879176.829888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879176.829902:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879176/real 1788879176] req@ffffa0618869ad80 x1875771511591296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879176.829914:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511591296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879176.829938:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879176.829942:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879176.829945:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879176.829951:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879176.829971:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879176.829983:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879176/real 1788879176] req@ffffa0618869bb80 x1875771511591424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879176.829995:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511591424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879176.830008:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879176.830012:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879176.830015:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879176.830022:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879176.830041:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879176.830051:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879176/real 1788879176] req@ffffa0618869a680 x1875771511591552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879176.830062:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511591552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879176.830070:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879176.830073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879176.830076:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879176.830082:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879176.830098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879176.830108:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879176/real 1788879176] req@ffffa06188699f80 x1875771511591680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879176.830117:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511591680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879176.830124:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879176.830127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879176.830131:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879177.469450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879177.469546:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879177.725493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879178.493486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879178.493596:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879178.749489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879179.517517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879179.517613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879179.773548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879180.541547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879180.541708:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879180.797476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879181.565440:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879181.565557:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879181.821498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879181.949532:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879181.949542:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879181.949547:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879181.949557:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4168 00000100:00080000:2.0:1788879181.949567:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879181.949574:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879181.949610:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511591808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879181.949637:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879181.949640:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879181.949641:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879181.949644:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4168 00000100:00080000:2.0:1788879181.949647:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879181.949650:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879181.949658:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879181.949667:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511591936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879181.949681:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511591808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879181.949688:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879181.949694:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879181.949697:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879181.949703:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4168 00000100:00080000:2.0:1788879181.949708:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879181.949715:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879181.949722:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879181.949739:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511592064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879181.949750:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879181.949754:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879181.949758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879181.949766:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4168 00000800:00000100:0.0:1788879181.949767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879181.949777:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879181.949784:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879181.949792:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879181.949809:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511592192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879181.949811:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879181/real 1788879181] req@ffffa061888af100 x1875771511591808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879181.949842:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511591808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879181.949855:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879181.949859:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879181.949861:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879181.949866:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788879181.949866:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879181.949874:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511591808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879181.949880:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4168 00000100:00080000:2.0:1788879181.949901:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879181.949907:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879181.949915:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879181.949921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879181.949935:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511592320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879181.949946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879181.949963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879181/real 1788879181] req@ffffa061888ae300 x1875771511591936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879181.949972:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511591936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879181.949980:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879181.949986:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879181.949989:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511591936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879181.949994:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879181.950008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879181.950014:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879181/real 1788879181] req@ffffa061888aea00 x1875771511592064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879181.950021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511592064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879181.950026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879181.950028:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879181.950029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879181.950033:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879181.950046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879181.950056:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879181/real 1788879181] req@ffffa061888afb80 x1875771511592192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879181.950068:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511592192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879181.950077:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879181.950080:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879181.950084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879181.950102:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879181.950119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879181.950128:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879181/real 1788879181] req@ffffa061888adc00 x1875771511592320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879181.950141:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511592320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879181.950147:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879181.950149:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879181.950151:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879182.589471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879182.589569:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879182.845492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879183.613503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879183.613629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879183.869460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879184.637474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879184.637563:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879184.893538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879185.661450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879185.661588:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879185.917452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879186.685482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879186.685531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879186.941473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879187.069594:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879187.069608:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879187.069616:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879187.069636:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4173 00000100:00080000:1.0:1788879187.069656:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879187.069670:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879187.069746:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511592448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879187.069819:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879187.069822:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879187.069824:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879187.069827:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4173 00000100:00080000:1.0:1788879187.069830:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879187.069833:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879187.069844:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879187.069872:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511592576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879187.069893:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879187.069897:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879187.069899:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879187.069902:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4173 00000100:00080000:1.0:1788879187.069907:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879187.069911:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879187.069917:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879187.069933:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511592704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879187.069935:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511592448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879187.069967:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879187.069970:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879187.069971:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879187.069976:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4173 00000100:00080000:1.0:1788879187.069979:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879187.069982:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879187.069987:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879187.069997:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511592832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879187.070004:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879187.070007:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879187.070009:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879187.070013:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4173 00000100:00080000:1.0:1788879187.070032:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879187.070035:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879187.070041:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879187.070046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879187.070065:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511592960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879187.070112:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879187/real 1788879187] req@ffffa0618869aa00 x1875771511592448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879187.070145:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511592448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879187.070163:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879187.070170:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879187.070177:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511592448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879187.070190:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879187.070223:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879187.070237:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879187/real 1788879187] req@ffffa0618869b800 x1875771511592576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879187.070249:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511592576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879187.070260:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879187.070263:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879187.070267:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879187.070297:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879187.070315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879187.070324:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879187/real 1788879187] req@ffffa06188699c00 x1875771511592704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879187.070335:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511592704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879187.070342:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879187.070345:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879187.070348:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879187.070354:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879187.070370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879187.070379:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879187/real 1788879187] req@ffffa06188698380 x1875771511592832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879187.070420:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511592832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879187.070428:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879187.070431:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879187.070435:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879187.070441:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879187.070459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879187.070469:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879187/real 1788879187] req@ffffa0618869b100 x1875771511592960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879187.070492:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511592960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879187.070500:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879187.070503:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879187.070506:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511592960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879187.709496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879187.709773:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879187.965494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879188.733508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879188.733676:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879188.989504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879189.757525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879189.757619:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879190.013492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879190.781502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879190.781586:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879191.037514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879191.805518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879191.805772:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879192.061535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879192.189560:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879192.189574:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879192.189580:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879192.189597:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4178 00000100:00080000:1.0:1788879192.189613:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879192.189626:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879192.189678:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511593088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879192.189716:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879192.189721:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879192.189723:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879192.189729:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4178 00000100:00080000:1.0:1788879192.189733:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879192.189739:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879192.189752:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879192.189769:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511593216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879192.189785:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879192.189792:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788879192.189793:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511593088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879192.189795:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879192.189810:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4178 00000100:00080000:1.0:1788879192.189846:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879192.189852:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879192.189861:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879192.189880:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511593344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879192.189894:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879192.189899:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879192.189902:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879192.189907:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4178 00000100:00080000:1.0:1788879192.189925:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879192.189930:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879192.189938:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879192.189948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879192.189962:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511593472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879192.189977:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879192.189982:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879192.189984:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879192.189991:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4178 00000100:00080000:1.0:1788879192.189996:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879192.190001:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879192.190008:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879192.190013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879192/real 1788879192] req@ffffa0618869b100 x1875771511593088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879192.190027:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511593600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879192.190056:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511593088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879192.190075:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879192.190082:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879192.190088:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511593088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879192.190119:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879192.190151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879192.190167:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879192/real 1788879192] req@ffffa06188698380 x1875771511593216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879192.190182:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511593216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879192.190193:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879192.190197:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879192.190202:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879192.190209:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879192.190228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879192.190239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879192/real 1788879192] req@ffffa06188699c00 x1875771511593344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879192.190253:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511593344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879192.190262:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879192.190266:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879192.190270:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879192.190276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879192.190294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879192.190305:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879192/real 1788879192] req@ffffa0618869b800 x1875771511593472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879192.190328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511593472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879192.190338:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879192.190341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879192.190345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879192.190352:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879192.190370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879192.190412:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879192/real 1788879192] req@ffffa0618869aa00 x1875771511593600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879192.190429:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511593600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879192.190439:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879192.190443:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879192.190447:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879192.829503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879192.829670:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879193.085538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879193.853519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879193.853668:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879194.109436:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879194.877565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879194.877646:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879195.133504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879195.901546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879195.901667:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879196.157533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879196.925588:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879196.925678:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879197.181499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879197.309534:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879197.309547:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879197.309553:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879197.309569:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4184 00000100:00080000:1.0:1788879197.309586:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879197.309598:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879197.309650:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511593728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879197.309687:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879197.309693:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879197.309696:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879197.309703:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4184 00000100:00080000:1.0:1788879197.309708:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879197.309717:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879197.309720:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511593728:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879197.309731:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879197.309753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511593856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879197.309769:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879197.309774:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879197.309776:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879197.309815:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4184 00000100:00080000:1.0:1788879197.309838:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879197.309847:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879197.309856:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879197.309865:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879197.309906:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511593984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879197.309916:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879197/real 1788879197] req@ffffa06188699f80 x1875771511593728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879197.309976:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879197.309984:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788879197.309986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511593728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879197.309989:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879197.310011:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4184 00000100:00080000:0.0:1788879197.310015:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879197.310024:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879197.310026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879197.310042:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511593728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879197.310045:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879197.310066:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879197.310094:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511594112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879197.310099:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593856:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879197.310132:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879197.310140:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879197.310145:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879197.310155:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4184 00000100:00080000:1.0:1788879197.310165:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879197.310171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879197.310181:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879197.310195:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879197.310214:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879197/real 1788879197] req@ffffa0618869a680 x1875771511593856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879197.310221:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511594240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879197.310296:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511593856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879197.310311:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879197.310317:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879197.310322:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879197.310331:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879197.310363:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879197.310378:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879197/real 1788879197] req@ffffa0618869bb80 x1875771511593984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879197.310429:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511593984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879197.310440:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879197.310444:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879197.310448:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511593984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879197.310468:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879197.310495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879197.310509:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879197/real 1788879197] req@ffffa0618869ad80 x1875771511594112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879197.310523:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511594112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879197.310533:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879197.310537:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879197.310541:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879197.310548:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879197.310566:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879197.310577:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879197/real 1788879197] req@ffffa06188698000 x1875771511594240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879197.310590:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511594240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879197.310600:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879197.310603:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879197.310607:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879197.949528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879197.949695:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879198.205479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879198.973487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879198.973602:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879199.229484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879199.997534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879199.997755:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879200.253504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879201.021558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879201.021798:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879201.277470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879202.045553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879202.045765:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879202.301601:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879202.429544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879202.429560:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879202.429568:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879202.429601:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4189 00000100:00080000:1.0:1788879202.429619:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879202.429632:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879202.429702:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511594368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879202.429765:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879202.429782:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879202.429786:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879202.429794:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4189 00000100:00080000:1.0:1788879202.429801:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879202.429808:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879202.429825:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511594368:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879202.429825:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879202.429857:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511594496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879202.429873:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879202.429879:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879202.429882:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879202.429887:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4189 00000100:00080000:1.0:1788879202.429915:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879202.429920:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879202.429927:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879202.429934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879202.429952:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511594624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879202.429969:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879202.429974:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879202.429977:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879202.429986:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4189 00000100:00080000:1.0:1788879202.429992:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879202.430001:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879202/real 1788879202] req@ffffa06188698000 x1875771511594368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879202.430017:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879202.430034:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879202.430046:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511594368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879202.430054:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511594752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879202.430087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879202.430089:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879202.430100:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788879202.430101:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879202.430109:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879202.430114:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511594368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879202.430121:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4189 00000100:00080000:1.0:1788879202.430131:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879202.430137:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879202.430143:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879202.430157:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511594880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879202.430167:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879202.430225:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879202.430253:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879202/real 1788879202] req@ffffa0618869ad80 x1875771511594496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879202.430280:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511594496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879202.430293:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879202.430298:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879202.430303:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879202.430312:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879202.430342:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879202.430357:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879202/real 1788879202] req@ffffa0618869bb80 x1875771511594624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879202.430382:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511594624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879202.430439:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879202.430443:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879202.430448:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879202.430456:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879202.430483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879202.430497:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879202/real 1788879202] req@ffffa0618869a680 x1875771511594752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879202.430512:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511594752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879202.430523:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879202.430527:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879202.430531:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879202.430566:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879202.430586:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879202.430598:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879202/real 1788879202] req@ffffa06188699f80 x1875771511594880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879202.430612:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511594880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879202.430630:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879202.430635:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879202.430639:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511594880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879203.069524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879203.069648:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879203.325527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879204.093522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879204.093635:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879204.349517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879205.117499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879205.117617:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879205.373500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879206.141511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879206.141671:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879206.397487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879207.165437:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879207.165561:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879207.421528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879207.549547:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879207.549560:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879207.549567:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879207.549583:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4194 00000100:00080000:2.0:1788879207.549598:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879207.549610:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879207.549663:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511595008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879207.549700:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879207.549705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879207.549728:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879207.549736:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4194 00000100:00080000:2.0:1788879207.549742:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879207.549749:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879207.549764:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879207.549785:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511595136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879207.549796:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511595008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879207.549805:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879207.549815:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879207.549818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879207.549824:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4194 00000100:00080000:2.0:1788879207.549828:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879207.549832:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879207.549838:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879207.549854:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511595264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879207.549867:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879207.549871:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879207.549874:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879207.549880:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4194 00000100:00080000:2.0:1788879207.549885:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879207.549890:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879207.549898:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879207.549913:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511595392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879207.549926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879207.549938:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879207.549943:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879207.549946:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879207.549954:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4194 00000100:00080000:2.0:1788879207.549960:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879207.549965:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879207.549971:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879207.549984:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511595520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879207.549993:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879207/real 1788879207] req@ffffa061888ae680 x1875771511595008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879207.550037:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511595008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879207.550059:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879207.550068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879207.550076:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511595008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879207.550118:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879207.550173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879207.550193:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879207/real 1788879207] req@ffffa061888ac380 x1875771511595136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879207.550209:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511595136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879207.550238:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879207.550242:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879207.550247:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879207.550255:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879207.550276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879207.550288:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879207/real 1788879207] req@ffffa061888ad880 x1875771511595264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879207.550302:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511595264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879207.550312:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879207.550316:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879207.550320:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879207.550326:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879207.550348:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879207.550377:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879207/real 1788879207] req@ffffa061888aed80 x1875771511595392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879207.550445:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511595392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879207.550463:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879207.550468:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879207.550473:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879207.550482:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879207.550513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879207.550528:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879207/real 1788879207] req@ffffa061888ac700 x1875771511595520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879207.550543:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511595520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879207.550564:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879207.550567:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879207.550571:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879208.189560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879208.189629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879208.445441:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879209.213535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879209.213636:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879209.469523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879210.237542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879210.237714:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879210.493541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879211.261532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879211.261758:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879211.517573:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879212.285521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879212.285568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879212.541497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879212.669535:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879212.669557:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879212.669562:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879212.669578:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4199 00000100:00080000:2.0:1788879212.669594:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879212.669606:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879212.669649:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511595648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879212.669693:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879212.669697:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879212.669699:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879212.669703:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4199 00000100:00080000:2.0:1788879212.669706:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879212.669710:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879212.669732:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879212.669747:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511595776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879212.669757:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879212.669760:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879212.669764:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879212.669775:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511595648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879212.669775:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4199 00000100:00080000:2.0:1788879212.669796:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879212.669805:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879212.669813:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879212.669825:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511595904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879212.669840:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879212.669842:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879212.669844:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879212.669849:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4199 00000100:00080000:2.0:1788879212.669851:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879212.669854:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879212.669857:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879212.669868:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511596032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879212.669876:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879212.669880:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879212.669881:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879212.669886:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4199 00000100:00080000:2.0:1788879212.669891:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879212.669896:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879212.669904:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879212.669916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879212.669924:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511596160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879212.669976:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879212/real 1788879212] req@ffffa061888ad180 x1875771511595648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879212.670002:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511595648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879212.670017:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879212.670022:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879212.670027:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511595648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879212.670079:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879212.670116:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879212.670128:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879212/real 1788879212] req@ffffa061888ace00 x1875771511595776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879212.670141:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511595776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879212.670150:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879212.670164:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879212.670168:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879212.670174:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879212.670192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879212.670201:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879212/real 1788879212] req@ffffa061888ac700 x1875771511595904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879212.670212:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511595904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879212.670221:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879212.670224:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879212.670227:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511595904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879212.670233:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879212.670249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879212.670263:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879212/real 1788879212] req@ffffa061888aed80 x1875771511596032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879212.670282:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511596032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879212.670295:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879212.670301:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879212.670307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879212.670318:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879212.670349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879212.670363:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879212/real 1788879212] req@ffffa061888ad880 x1875771511596160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879212.670386:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511596160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879212.670394:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879212.670427:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879212.670431:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879213.309493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879213.309588:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879213.565549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879214.333499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879214.333583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879214.589551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879215.357537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879215.359532:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879215.613519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879216.381487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879216.381686:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879216.637505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879217.405459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879217.405532:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879217.661580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879217.789537:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879217.789551:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879217.789557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879217.789571:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4204 00000100:00080000:3.0:1788879217.789587:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879217.789601:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879217.789692:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511596288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879217.789730:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879217.789738:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879217.789740:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879217.789747:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4204 00000100:00080000:3.0:1788879217.789754:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879217.789761:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879217.789775:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879217.789792:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511596288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879217.789795:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511596416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879217.789818:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879217.789839:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879217.789842:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879217.789848:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4204 00000100:00080000:3.0:1788879217.789852:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879217.789857:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879217.789862:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879217.789878:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511596544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879217.789902:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879217.789907:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879217.789909:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879217.789915:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4204 00000100:00080000:3.0:1788879217.789920:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879217.789924:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879217.789949:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879217.789956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879217.789980:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511596672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879217.790019:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879217.790025:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879217.790029:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879217.790037:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4204 00000100:00100000:0.0:1788879217.790044:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879217/real 1788879217] req@ffffa06188872300 x1875771511596288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879217.790047:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879217.790063:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879217.790092:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879217.790111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511596288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879217.790114:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511596800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879217.790146:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879217.790154:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879217.790160:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511596288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879217.790191:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879217.790219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879217.790236:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879217/real 1788879217] req@ffffa06188873b80 x1875771511596416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879217.790257:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511596416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879217.790272:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879217.790278:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879217.790284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879217.790296:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879217.790329:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879217.790346:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879217/real 1788879217] req@ffffa06188870700 x1875771511596544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879217.790386:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511596544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879217.790416:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879217.790440:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879217.790447:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879217.790504:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879217.790538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879217.790554:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879217/real 1788879217] req@ffffa06188872d80 x1875771511596672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879217.790585:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511596672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879217.790597:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879217.790601:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879217.790605:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879217.790613:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879217.790677:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879217.790694:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879217/real 1788879217] req@ffffa06188873800 x1875771511596800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879217.790726:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511596800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879217.790735:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879217.790738:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879217.790742:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511596800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879218.429475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879218.429566:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879218.685518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879219.453539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879219.453716:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879219.709466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879220.477527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879220.477697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879220.733516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879221.501562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879221.501594:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879221.757482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879222.525495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879222.525646:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879222.781503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788879222.909540:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879222.909552:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788879222.909557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879222.909571:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4209 00000100:00080000:0.0:1788879222.909584:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879222.909594:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879222.909715:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511596928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879222.909763:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879222.909770:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788879222.909774:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879222.909781:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4209 00000100:00080000:0.0:1788879222.909805:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879222.909814:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879222.909826:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879222.909827:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511596928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879222.909843:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511597056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879222.909857:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879222.909880:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788879222.909883:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879222.909889:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4209 00000100:00080000:0.0:1788879222.909895:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879222.909901:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879222.909919:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788879222.909932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879222.909959:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511597184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879222.909970:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879222.909973:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788879222.909975:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879222.909979:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4209 00000100:00080000:0.0:1788879222.909984:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879222.909988:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879222.909994:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879222.910000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879222/real 1788879222] req@ffffa06198afea00 x1875771511596928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879222.910008:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511597312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879222.910042:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879222.910046:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879222.910048:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879222.910073:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4209 00000100:00080000:0.0:1788879222.910076:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788879222.910079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511596928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879222.910082:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879222.910091:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879222.910101:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511597440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879222.910113:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879222.910120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879222.910125:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511596928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879222.910157:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879222.910218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879222.910239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879222/real 1788879222] req@ffffa06198afc000 x1875771511597056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879222.910268:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511597056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879222.910280:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879222.910285:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879222.910290:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879222.910299:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879222.910324:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879222.910363:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879222/real 1788879222] req@ffffa06198afc700 x1875771511597184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879222.910373:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511597184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879222.910415:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879222.910420:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879222.910423:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879222.910431:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879222.910453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879222.910476:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879222/real 1788879222] req@ffffa06198afdc00 x1875771511597312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879222.910493:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511597312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879222.910517:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879222.910520:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879222.910522:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879222.910528:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879222.910543:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879222.910551:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879222/real 1788879222] req@ffffa06198afdf80 x1875771511597440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879222.910560:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511597440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879222.910567:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879222.910569:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879222.910572:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879223.549494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879223.549585:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879223.805445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879224.573518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879224.573564:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879224.829497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879225.597545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879225.597634:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879225.853494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879226.621488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879226.621591:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879226.877536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879227.645495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879227.645591:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879227.901505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879228.029694:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879228.029708:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879228.029715:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879228.029731:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4214 00000100:00080000:2.0:1788879228.029745:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879228.029757:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879228.029824:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511597568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879228.029858:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879228.029863:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879228.029866:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879228.029873:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4214 00000100:00080000:2.0:1788879228.029881:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879228.029907:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879228.029924:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879228.029946:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511597568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879228.029957:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511597696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879228.029996:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879228.030000:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879228.030003:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879228.030009:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4214 00000100:00080000:2.0:1788879228.030017:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879228.030022:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879228.030030:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788879228.030038:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879228.030054:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511597824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879228.030067:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879228.030071:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879228.030074:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879228.030079:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4214 00000100:00080000:2.0:1788879228.030084:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879228.030089:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879228.030096:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879228.030109:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511597952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879228.030117:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879228/real 1788879228] req@ffffa061888ac380 x1875771511597568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879228.030147:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788879228.030151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511597568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879228.030155:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879228.030161:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879228.030175:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4214 00000100:00080000:2.0:1788879228.030185:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879228.030196:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879228.030207:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1788879228.030210:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879228.030220:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879228.030226:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511598080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879228.030237:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511597568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879228.030301:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879228.030349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879228.030374:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879228/real 1788879228] req@ffffa061888ae680 x1875771511597696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879228.030441:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511597696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879228.030452:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879228.030456:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879228.030460:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879228.030468:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788879228.030497:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879228.030512:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879228/real 1788879228] req@ffffa061888adc00 x1875771511597824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879228.030535:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511597824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879228.030550:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879228.030556:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879228.030562:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879228.030597:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879228.030638:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879228.030655:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879228/real 1788879228] req@ffffa061888afb80 x1875771511597952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879228.030669:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511597952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879228.030680:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879228.030683:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879228.030687:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511597952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879228.030694:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788879228.030715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879228.030725:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879228/real 1788879228] req@ffffa061888aea00 x1875771511598080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879228.030738:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511598080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879228.030747:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879228.030750:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879228.030753:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879228.669516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879228.669670:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879228.925549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879229.693441:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879229.693516:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879229.949455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879230.717528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879230.717614:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879230.973470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879231.741513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879231.741694:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879231.997507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879232.765472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879232.765560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879233.021495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879233.149520:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879233.149531:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879233.149549:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879233.149561:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4219 00000100:00080000:3.0:1788879233.149573:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879233.149582:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879233.149643:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511598208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879233.149674:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879233.149677:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879233.149678:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879233.149682:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4219 00000100:00080000:3.0:1788879233.149685:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879233.149689:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879233.149698:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879233.149710:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511598336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879233.149718:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511598208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879233.149720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879233.149727:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879233.149730:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879233.149733:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4219 00000100:00080000:3.0:1788879233.149735:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879233.149738:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879233.149741:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879233.149749:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511598464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879233.149760:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879233.149762:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879233.149764:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879233.149766:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4219 00000100:00080000:3.0:1788879233.149769:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879233.149771:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879233.149774:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879233.149792:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511598592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879233.149798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879233.149801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879233.149806:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879233.149812:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4219 00000800:00000100:0.0:1788879233.149815:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879233.149816:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879233.149821:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879233.149825:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879233.149834:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511598720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879233.149859:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879233/real 1788879233] req@ffffa06188871c00 x1875771511598208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879233.149882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511598208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879233.149895:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879233.149899:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879233.149904:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511598208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879233.149929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879233.149955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879233.149970:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879233/real 1788879233] req@ffffa06188870a80 x1875771511598336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879233.149986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511598336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879233.149996:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879233.150001:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879233.150005:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879233.150012:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879233.150034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879233.150066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879233/real 1788879233] req@ffffa06188870380 x1875771511598464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879233.150087:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511598464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879233.150101:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879233.150107:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879233.150113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879233.150123:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879233.150164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879233.150175:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879233/real 1788879233] req@ffffa06188871880 x1875771511598592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879233.150190:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511598592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879233.150200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879233.150204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879233.150225:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879233.150235:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879233.150266:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879233.150294:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879233/real 1788879233] req@ffffa06188873800 x1875771511598720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879233.150308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511598720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879233.150317:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879233.150321:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879233.150324:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879233.789488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879233.789600:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879234.045475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879234.813589:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879234.813666:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879235.069598:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879235.837576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879235.837624:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879236.093593:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879236.861501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879236.861664:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879237.117536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879237.885537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879237.885650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879238.141520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879238.269593:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879238.269607:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879238.269614:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879238.269630:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4224 00000100:00080000:1.0:1788879238.269646:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879238.269658:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879238.269713:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511598848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879238.269754:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879238.269760:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879238.269763:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879238.269771:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4224 00000100:00080000:1.0:1788879238.269777:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879238.269784:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879238.269798:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879238.269816:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511598976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879238.269828:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511598848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879238.269850:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879238.269858:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879238.269862:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879238.269872:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4224 00000100:00080000:1.0:1788879238.269881:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879238.269890:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879238.269902:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879238.269929:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511599104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879238.269960:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879238.269966:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879238.269969:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879238.269976:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4224 00000100:00100000:0.0:1788879238.269986:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598976:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879238.269986:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879238.270002:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879238.270011:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879238.270035:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511599232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879238.270051:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879238.270052:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599104:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879238.270070:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879238.270075:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879238.270082:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4224 00000100:00080000:1.0:1788879238.270102:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879238.270106:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599232:192.168.203.111@tcp:8:kworker.0 00000100:00080000:1.0:1788879238.270114:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879238.270125:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879238.270145:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511599360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879238.270201:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879238.909566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879238.909663:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879239.165485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879239.933547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879239.933717:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879240.189473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879240.957542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879240.957640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879241.213529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:3.0:1788879241.341751:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:3.0:1788879241.341806:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788879241.341826:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788879241.341867:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788879241.341883:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788879241.341898:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:3.0:1788879241.341911:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879241.341930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879238/real 1788879241] req@ffffa06188698380 x1875771511599232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879241.341965:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511599232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879241.341991:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879241.341997:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879241.342001:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879241.342009:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879238/real 1788879241] req@ffffa0618869b800 x1875771511598976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879241.342021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511598976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879241.342029:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879241.342032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879241.342035:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511598976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879241.342040:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879238/real 1788879241] req@ffffa06188699c00 x1875771511599104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879241.342057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511599104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879241.342068:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879241.342072:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879241.342076:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879241.342086:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879238/real 1788879241] req@ffffa0618869aa00 x1875771511598848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879241.342103:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511598848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879241.342115:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879241.342120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879241.342125:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511598848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879241.342161:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879238/real 1788879241] req@ffffa0618869b100 x1875771511599360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879241.342178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511599360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879241.342188:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879241.342192:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879241.342197:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879241.981496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879241.981564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879242.237582:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879243.005522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879243.005676:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879243.261536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879243.389577:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879243.389590:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879243.389597:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879243.389613:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4230 00000100:00080000:1.0:1788879243.389631:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879243.389645:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879243.389695:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511599488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879243.389731:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879243.389736:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879243.389739:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879243.389744:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4230 00000100:00080000:1.0:1788879243.389749:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879243.389755:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879243.389770:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879243.389787:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511599616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879243.389800:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511599488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879243.389806:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879243.389820:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879243.389824:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879243.389830:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4230 00000100:00080000:1.0:1788879243.389834:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879243.389841:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879243.389850:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879243.389878:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511599744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879243.389899:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879243.389906:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879243.389911:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879243.389918:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4230 00000100:00080000:1.0:1788879243.389927:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879243.389937:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879243.389946:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879243.389959:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879243.389988:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511599872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879243.390008:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879243.390014:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879243/real 1788879243] req@ffffa0618869b100 x1875771511599488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879243.390018:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879243.390039:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879243.390054:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4230 00000100:00080000:1.0:1788879243.390065:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879243.390067:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511599488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879243.390084:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879243.390106:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879243.390112:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879243.390127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879243.390139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511599488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879243.390175:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511600000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879243.390198:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879243.390258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879243.390288:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879243/real 1788879243] req@ffffa0618869aa00 x1875771511599616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879243.390317:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511599616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879243.390335:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879243.390341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879243.390349:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879243.390361:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879243.390449:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879243.390511:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879243/real 1788879243] req@ffffa06188699c00 x1875771511599744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879243.390537:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511599744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879243.390574:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879243.390582:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879243.390589:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879243.390601:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879243.390647:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879243.390672:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879243/real 1788879243] req@ffffa0618869b800 x1875771511599872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879243.390696:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511599872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879243.390712:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879243.390718:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879243.390724:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511599872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879243.390756:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879243.390798:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879243.390822:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879243/real 1788879243] req@ffffa06188698380 x1875771511600000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879243.390845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511600000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879243.390862:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879243.390869:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879243.390875:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879244.029552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879244.029643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879244.285528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879245.053577:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879245.053597:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879245.309477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879246.077571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879246.077658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879246.333511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879247.101520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879247.101638:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879247.357526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879248.125553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879248.125663:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879248.381498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879248.509576:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879248.509589:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879248.509596:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879248.509610:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4235 00000100:00080000:2.0:1788879248.509626:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879248.509637:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879248.509695:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511600128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879248.509759:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879248.509767:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879248.509771:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879248.509782:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4235 00000100:00080000:2.0:1788879248.509791:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879248.509799:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879248.509814:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879248.509820:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511600128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879248.509840:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511600256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879248.509859:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879248.509863:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879248.509866:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879248.509870:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4235 00000100:00080000:2.0:1788879248.509875:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879248.509879:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879248.509886:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879248.509901:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511600384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879248.509914:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879248.509919:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879248.509921:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879248.509925:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4235 00000100:00080000:2.0:1788879248.509931:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879248.509940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879248.509944:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879248.509954:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879248.509971:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511600512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879248.509984:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879248.509989:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879248.509991:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879248.509997:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4235 00000100:00080000:2.0:1788879248.510001:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879248.510008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879248/real 1788879248] req@ffffa061888ae300 x1875771511600128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879248.510010:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879248.510021:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879248.510040:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511600640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879248.510072:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511600128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879248.510096:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879248.510103:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879248.510108:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511600128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879248.510144:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879248.510192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879248.510217:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879248/real 1788879248] req@ffffa061888af100 x1875771511600256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879248.510242:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511600256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879248.510254:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879248.510258:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879248.510262:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879248.510286:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879248.510323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879248.510339:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879248/real 1788879248] req@ffffa061888ad500 x1875771511600384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879248.510353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511600384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879248.510372:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879248.510377:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879248.510381:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879248.510418:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879248.510476:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879248.510493:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879248/real 1788879248] req@ffffa061888ac000 x1875771511600512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879248.510508:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511600512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879248.510519:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879248.510523:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879248.510527:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879248.510535:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879248.510555:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879248.510567:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879248/real 1788879248] req@ffffa061888adf80 x1875771511600640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879248.510581:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511600640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879248.510590:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879248.510593:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879248.510599:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879249.149528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879249.149702:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879249.405579:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879250.173522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879250.173636:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879250.429566:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879251.197501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879251.197623:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879251.453565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879252.221527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879252.221656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879252.477536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879253.245491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879253.245597:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879253.501489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879253.629501:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879253.629509:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879253.629514:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879253.629523:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4240 00000100:00080000:3.0:1788879253.629533:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879253.629540:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879253.629576:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511600768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879253.629597:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879253.629599:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879253.629601:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879253.629604:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4240 00000100:00080000:3.0:1788879253.629612:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879253.629615:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879253.629624:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879253.629633:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511600768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879253.629635:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511600896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879253.629647:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879253.629649:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879253.629650:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879253.629653:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4240 00000100:00080000:3.0:1788879253.629656:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879253.629659:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879253.629663:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879253.629672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511601024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879253.629678:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879253.629681:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879253.629683:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879253.629685:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4240 00000100:00080000:3.0:1788879253.629688:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879253.629690:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879253.629693:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879253.629701:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511601152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879253.629709:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879253.629711:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879253.629714:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879253.629716:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879253.629719:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4240 00000100:00080000:3.0:1788879253.629722:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879253.629725:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879253.629727:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879253.629736:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511601280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879253.629747:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879253/real 1788879253] req@ffffa06188872d80 x1875771511600768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879253.629766:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511600768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879253.629776:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879253.629779:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879253.629782:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511600768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879253.629802:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879253.629828:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879253.629836:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879253/real 1788879253] req@ffffa06188870700 x1875771511600896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879253.629843:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511600896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879253.629849:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879253.629852:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879253.629854:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511600896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879253.629858:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879253.629869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879253.629874:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879253/real 1788879253] req@ffffa06188873b80 x1875771511601024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879253.629882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511601024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879253.629887:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879253.629888:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879253.629890:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879253.629894:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879253.629904:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879253.629910:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879253/real 1788879253] req@ffffa06188872300 x1875771511601152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879253.629917:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511601152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879253.629923:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879253.629924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879253.629926:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879253.629929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879253.629939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879253.629945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879253/real 1788879253] req@ffffa06188872a00 x1875771511601280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879253.629965:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511601280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879253.629973:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879253.629977:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879253.629980:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879254.269520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879254.269620:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879254.525548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879255.293607:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879255.293728:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879255.549512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879256.317454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879256.317515:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879256.573514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879257.341503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879257.341683:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879257.597478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879258.365487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879258.365614:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879258.621536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879258.749569:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879258.749584:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879258.749592:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879258.749609:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4245 00000100:00080000:3.0:1788879258.749631:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879258.749644:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879258.749702:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511601408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879258.749744:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879258.749750:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879258.749752:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879258.749758:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4245 00000100:00080000:3.0:1788879258.749763:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879258.749768:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879258.749783:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879258.749803:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511601536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879258.749824:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511601408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879258.749827:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879258.749846:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879258.749858:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879258.749868:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4245 00000100:00080000:3.0:1788879258.749871:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879258.749876:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879258.749884:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879258.749902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511601664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879258.749914:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879258.749920:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879258.749923:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879258.749929:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4245 00000100:00080000:3.0:1788879258.749933:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879258.749937:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879258.749944:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879258.749983:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511601792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879258.749984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879258.749999:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879258.750003:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879258.750005:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879258.750009:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4245 00000100:00080000:3.0:1788879258.750012:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879258.750015:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879258.750018:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879258.750027:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511601920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879258.750084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879258/real 1788879258] req@ffffa06188872680 x1875771511601408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879258.750112:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511601408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879258.750133:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879258.750142:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879258.750161:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511601408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879258.750211:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879258.750276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879258.750296:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879258/real 1788879258] req@ffffa06188873100 x1875771511601536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879258.750314:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511601536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879258.750331:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879258.750339:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879258.750345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879258.750429:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879258.750471:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879258.750500:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879258/real 1788879258] req@ffffa06188871500 x1875771511601664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879258.750516:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511601664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879258.750527:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879258.750531:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879258.750535:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879258.750544:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879258.750578:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879258.750591:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879258/real 1788879258] req@ffffa06188873480 x1875771511601792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879258.750615:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511601792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879258.750625:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879258.750628:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879258.750632:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879258.750639:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879258.750658:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879258.750670:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879258/real 1788879258] req@ffffa06188872a00 x1875771511601920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879258.750684:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511601920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879258.750694:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879258.750697:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879258.750701:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511601920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879259.389511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879259.389617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879259.645483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879260.413545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879260.413653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879260.669613:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879261.437530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879261.437628:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879261.693462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879262.461502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879262.461640:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879262.717483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879263.485534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879263.485672:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879263.741467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879263.869511:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879263.869524:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879263.869531:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879263.869545:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4250 00000100:00080000:2.0:1788879263.869560:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879263.869570:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879263.869645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511602048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879263.869684:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879263.869691:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879263.869694:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879263.869699:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4250 00000100:00080000:2.0:1788879263.869704:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879263.869710:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879263.869732:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879263.869748:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511602048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879263.869752:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511602176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879263.869773:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879263.869776:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879263.869778:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879263.869782:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4250 00000100:00080000:2.0:1788879263.869803:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879263.869806:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879263.869810:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879263.869823:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511602304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879263.869831:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879263.869835:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879263.869836:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879263.869839:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4250 00000100:00080000:2.0:1788879263.869842:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879263.869845:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879263.869850:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879263.869858:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511602432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879263.869867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879263.869874:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879263.869877:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879263.869879:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879263.869883:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4250 00000100:00080000:2.0:1788879263.869893:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879263.869897:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879263.869901:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879263.869910:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511602560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879263.869925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879263/real 1788879263] req@ffffa061888aca80 x1875771511602048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879263.869952:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511602048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879263.869966:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879263.869972:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879263.869978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511602048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879263.870010:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879263.870039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879263.870073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879263/real 1788879263] req@ffffa061888af480 x1875771511602176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879263.870094:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511602176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879263.870108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879263.870113:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879263.870119:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879263.870129:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879263.870160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879263.870178:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879263/real 1788879263] req@ffffa061888adf80 x1875771511602304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879263.870198:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511602304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879263.870235:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879263.870241:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879263.870245:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879263.870255:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879263.870283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879263.870298:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879263/real 1788879263] req@ffffa061888ac000 x1875771511602432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879263.870349:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511602432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879263.870360:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879263.870365:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879263.870369:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879263.870376:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879263.870430:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879263.870445:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879263/real 1788879263] req@ffffa061888ad500 x1875771511602560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879263.870460:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511602560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879263.870470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879263.870474:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879263.870480:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879264.509456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879264.509596:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879264.765464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879265.533517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879265.533702:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879265.789543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879266.557524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879266.557624:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879266.813533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879267.581561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879267.581601:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879267.837488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879268.605514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879268.605606:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879268.861488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879268.989585:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879268.989598:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879268.989605:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879268.989620:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4255 00000100:00080000:3.0:1788879268.989636:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879268.989647:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879268.989710:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511602688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879268.989747:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879268.989754:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879268.989758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879268.989765:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4255 00000100:00080000:3.0:1788879268.989773:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879268.989783:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879268.989801:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879268.989815:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511602688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879268.989832:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511602816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879268.989856:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879268.989863:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879268.989866:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879268.989872:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4255 00000100:00080000:3.0:1788879268.989878:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879268.989884:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879268.989892:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879268.989909:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511602944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879268.989927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879268.989944:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879268.989949:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879268.989952:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879268.989959:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4255 00000100:00080000:3.0:1788879268.989964:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879268.989992:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879268.990000:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879268.990012:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879268/real 1788879268] req@ffffa06188872300 x1875771511602688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879268.990022:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511603072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879268.990059:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879268.990064:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879268.990073:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879268.990074:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511602688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879268.990082:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4255 00000100:00080000:3.0:1788879268.990091:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879268.990104:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879268.990111:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788879268.990113:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879268.990126:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879268.990136:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511602688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879268.990157:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511603200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879268.990194:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879268.990228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879268.990263:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879268/real 1788879268] req@ffffa06188873b80 x1875771511602816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879268.990279:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511602816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879268.990314:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879268.990319:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879268.990324:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879268.990331:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879268.990358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879268.990372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879268/real 1788879268] req@ffffa06188870700 x1875771511602944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879268.990424:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511602944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879268.990440:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879268.990446:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879268.990452:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511602944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879268.990465:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879268.990505:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879268.990524:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879268/real 1788879268] req@ffffa06188872d80 x1875771511603072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879268.990538:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511603072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879268.990548:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879268.990552:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879268.990556:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879268.990563:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879268.990586:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879268.990597:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879268/real 1788879268] req@ffffa06188873800 x1875771511603200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879268.990611:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511603200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879268.990620:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879268.990623:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879268.990626:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879269.629500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879269.629622:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879269.885530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879270.653448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879270.653553:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879270.909528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879271.677538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879271.677691:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879271.933509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879272.701532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879272.701695:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879272.957540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879273.725474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879273.725636:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879273.981471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879274.109512:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879274.109526:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879274.109533:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879274.109549:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4260 00000100:00080000:3.0:1788879274.109566:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879274.109576:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879274.109638:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511603328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879274.109676:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879274.109682:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879274.109685:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879274.109692:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4260 00000100:00080000:3.0:1788879274.109697:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879274.109703:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879274.109718:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879274.109736:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511603456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879274.109746:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511603328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879274.109758:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879274.109764:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879274.109766:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879274.109772:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4260 00000100:00080000:3.0:1788879274.109776:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879274.109780:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879274.109787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879274.109805:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511603584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879274.109823:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879274.109832:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879274.109836:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879274.109845:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4260 00000800:00000100:0.0:1788879274.109853:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879274.109859:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879274.109872:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879274.109884:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879274.109918:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511603712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879274.109937:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879274.109938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879274/real 1788879274] req@ffffa06188871880 x1875771511603328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879274.109952:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879274.109960:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879274.109975:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4260 00000100:00080000:3.0:1788879274.109984:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879274.109990:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879274.109998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511603328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879274.110002:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879274.110026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788879274.110033:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511603840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879274.110040:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879274.110057:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511603328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879274.110093:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879274.110125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879274.110142:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879274/real 1788879274] req@ffffa06188870380 x1875771511603456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879274.110157:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511603456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879274.110168:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879274.110172:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879274.110177:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879274.110184:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879274.110202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879274.110214:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879274/real 1788879274] req@ffffa06188870a80 x1875771511603584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879274.110228:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511603584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879274.110237:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879274.110241:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879274.110244:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879274.110278:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879274.110301:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879274.110314:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879274/real 1788879274] req@ffffa06188871c00 x1875771511603712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879274.110328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511603712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879274.110339:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879274.110343:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879274.110347:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879274.110354:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879274.110372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879274.110414:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879274/real 1788879274] req@ffffa06188873800 x1875771511603840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879274.110429:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511603840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879274.110440:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879274.110443:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879274.110447:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511603840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879274.749541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879274.749626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879275.005482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879275.773536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879275.773612:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879276.029519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788879276.797593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879276.797650:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788879277.053534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879277.821525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879277.821584:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879278.077504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879278.845510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879278.845604:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879279.101517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879279.229579:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879279.229591:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879279.229597:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879279.229610:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4265 00000100:00080000:1.0:1788879279.229623:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879279.229633:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879279.229690:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511603968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879279.229736:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879279.229741:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879279.229743:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879279.229749:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4265 00000100:00080000:1.0:1788879279.229753:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879279.229758:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879279.229771:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879279.229796:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511603968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879279.229798:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511604096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879279.229829:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879279.229836:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879279.229841:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879279.229848:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4265 00000100:00080000:1.0:1788879279.229857:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879279.229866:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879279.229877:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879279.229912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879279.229914:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511604224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879279.229945:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879279.229950:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879279.229952:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879279.229959:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4265 00000100:00100000:0.0:1788879279.229970:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879279/real 1788879279] req@ffffa06188699f80 x1875771511603968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879279.229974:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879279.229988:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879279.230001:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879279.230018:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511603968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879279.230021:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511604352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879279.230065:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879279.230072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879279.230075:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879279.230086:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788879279.230099:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879279.230100:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511603968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879279.230118:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4265 00000100:00080000:1.0:1788879279.230127:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879279.230137:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879279.230147:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879279.230158:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879279.230172:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511604480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879279.230240:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879279.230275:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879279/real 1788879279] req@ffffa0618869a680 x1875771511604096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879279.230291:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511604096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879279.230319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879279.230326:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879279.230332:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879279.230344:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879279.230420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879279.230441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879279/real 1788879279] req@ffffa0618869bb80 x1875771511604224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879279.230456:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511604224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879279.230466:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879279.230470:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879279.230474:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879279.230482:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879279.230509:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879279.230520:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879279/real 1788879279] req@ffffa0618869ad80 x1875771511604352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879279.230534:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511604352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879279.230562:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879279.230566:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879279.230569:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879279.230591:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879279.230613:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879279.230625:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879279/real 1788879279] req@ffffa06188698000 x1875771511604480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879279.230639:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511604480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879279.230673:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879279.230676:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879279.230679:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879279.869519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879279.869636:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879280.125507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879280.893482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879280.893568:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879281.149467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879281.917507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879281.917589:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879282.173555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879282.941552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879282.941710:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879283.197503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879283.965505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879283.965661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879284.221501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879284.349515:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879284.349530:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879284.349537:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879284.349553:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4271 00000100:00080000:1.0:1788879284.349571:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879284.349583:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879284.349637:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511604608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879284.349675:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879284.349682:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879284.349686:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879284.349695:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4271 00000100:00080000:1.0:1788879284.349706:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879284.349715:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879284.349734:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511604608:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879284.349735:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879284.349775:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511604736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879284.349798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879284.349804:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879284.349808:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879284.349817:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4271 00000100:00080000:1.0:1788879284.349840:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879284.349853:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879284.349856:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879284.349872:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879284.349904:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511604864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879284.349916:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879284/real 1788879284] req@ffffa06188698000 x1875771511604608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879284.349927:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879284.349941:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879284.349947:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879284.349957:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4271 00000100:00080000:1.0:1788879284.349964:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879284.349971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511604608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879284.349978:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879284.349992:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879284.349999:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879284.350010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879284.350016:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511604992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879284.350024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511604608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879284.350048:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879284.350052:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879284.350055:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879284.350061:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4271 00000100:00080000:1.0:1788879284.350066:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879284.350071:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879284.350077:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604736:192.168.203.111@tcp:38:kworker.0 00000008:00100000:1.0:1788879284.350082:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879284.350104:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511605120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879284.350135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879284.350156:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879284/real 1788879284] req@ffffa0618869ad80 x1875771511604736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879284.350173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511604736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879284.350214:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879284.350219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879284.350224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879284.350235:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879284.350274:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879284.350298:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879284/real 1788879284] req@ffffa0618869bb80 x1875771511604864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879284.350324:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511604864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879284.350341:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879284.350345:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879284.350350:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879284.350358:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879284.350422:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879284.350438:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879284/real 1788879284] req@ffffa0618869a680 x1875771511604992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879284.350454:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511604992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879284.350464:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879284.350468:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879284.350472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511604992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879284.350492:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879284.350522:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879284.350543:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879284/real 1788879284] req@ffffa06188699f80 x1875771511605120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879284.350567:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511605120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879284.350598:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879284.350605:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879284.350611:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879284.989510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879284.989627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879285.245477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879286.013519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879286.013625:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879286.269505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879287.037513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879287.037605:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879287.293499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879288.061522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879288.061642:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879288.317518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879289.085480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879289.085642:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879289.341545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879289.469652:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879289.469684:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879289.469691:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879289.469707:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4276 00000100:00080000:2.0:1788879289.469725:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879289.469736:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879289.469797:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511605248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879289.469853:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879289.469863:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879289.469867:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879289.469877:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4276 00000100:00080000:2.0:1788879289.469890:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879289.469901:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879289.469922:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879289.469937:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511605248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879289.469953:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511605376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879289.469973:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879289.469979:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879289.469983:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879289.470004:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4276 00000100:00080000:2.0:1788879289.470011:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879289.470017:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879289.470024:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879289.470041:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511605504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879289.470075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879289.470093:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879289.470099:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879289.470102:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879289.470111:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4276 00000100:00080000:2.0:1788879289.470129:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879289.470137:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879289.470168:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879289.470182:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879289/real 1788879289] req@ffffa061888af100 x1875771511605248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879289.470218:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511605632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879289.470233:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511605248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879289.470328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879289.470337:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788879289.470338:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879289.470352:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788879289.470353:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511605248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879289.470360:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879289.470373:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4276 00000100:00080000:2.0:1788879289.470379:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879289.470385:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879289.470443:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605376:192.168.203.111@tcp:38:kworker.0 00000008:00100000:2.0:1788879289.470458:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879289.470490:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511605760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879289.470491:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879289.470524:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879289/real 1788879289] req@ffffa061888ae300 x1875771511605376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879289.470545:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511605376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879289.470559:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879289.470565:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879289.470569:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879289.470578:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879289.470599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879289.470623:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879289/real 1788879289] req@ffffa061888aea00 x1875771511605504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879289.470637:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511605504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879289.470647:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879289.470650:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879289.470654:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879289.470661:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879289.470691:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879289.470703:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879289/real 1788879289] req@ffffa061888afb80 x1875771511605632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879289.470721:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511605632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879289.470738:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879289.470744:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879289.470751:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879289.470785:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879289.470829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879289.470849:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879289/real 1788879289] req@ffffa061888adc00 x1875771511605760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879289.470864:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511605760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879289.470875:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879289.470879:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879289.470883:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511605760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879290.109596:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879290.109728:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879290.365546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879291.133513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879291.133649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879291.389578:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879292.157531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879292.157626:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879292.413543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879293.181457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879293.181525:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879293.437547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879294.205529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879294.205634:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879294.461522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879294.589552:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879294.589567:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879294.589574:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879294.589593:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4281 00000100:00080000:2.0:1788879294.589611:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879294.589621:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879294.589693:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511605888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879294.589730:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879294.589734:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879294.589751:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879294.589757:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4281 00000100:00080000:2.0:1788879294.589760:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879294.589764:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879294.589778:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879294.589791:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511606016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879294.589824:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879294.589830:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788879294.589831:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511605888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879294.589834:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879294.589854:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4281 00000100:00080000:2.0:1788879294.589860:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879294.589864:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879294.589871:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879294.589888:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511606144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879294.589898:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879294.589902:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879294.589905:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879294.589911:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4281 00000100:00080000:2.0:1788879294.589919:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879294.589922:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879294.589927:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879294.589939:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511606272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879294.589949:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879294.589953:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879294.589960:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879294.589965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879294.589983:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4281 00000100:00080000:2.0:1788879294.589990:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879294.589996:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879294.590019:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879294.590038:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511606400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879294.590068:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879294/real 1788879294] req@ffffa061888ae680 x1875771511605888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879294.590099:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511605888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879294.590148:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879294.590157:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879294.590166:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511605888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879294.590212:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879294.590270:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879294.590298:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879294/real 1788879294] req@ffffa061888ac380 x1875771511606016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879294.590325:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511606016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879294.590354:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879294.590362:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879294.590369:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879294.590422:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879294.590457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879294.590475:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879294/real 1788879294] req@ffffa061888ad880 x1875771511606144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879294.590490:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511606144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879294.590501:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879294.590506:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879294.590510:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879294.590518:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879294.590540:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879294.590553:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879294/real 1788879294] req@ffffa061888aed80 x1875771511606272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879294.590567:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511606272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879294.590578:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879294.590582:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879294.590586:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879294.590594:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879294.590615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879294.590626:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879294/real 1788879294] req@ffffa061888ac700 x1875771511606400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879294.590640:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511606400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879294.590650:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879294.590654:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879294.590658:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879295.229455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879295.229547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879295.485516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879296.253541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879296.253685:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879296.509553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879297.277493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879297.277626:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879297.533522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879298.301518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879298.301760:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879298.557557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879299.325503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879299.325713:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879299.581549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879299.709508:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879299.709525:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879299.709533:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879299.709550:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4286 00000100:00080000:2.0:1788879299.709568:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879299.709582:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879299.709654:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511606528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879299.709694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879299.709701:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879299.709705:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879299.709713:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4286 00000100:00080000:2.0:1788879299.709721:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879299.709730:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879299.709746:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879299.709748:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511606528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879299.709776:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511606656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879299.709810:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879299.709818:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879299.709821:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879299.709829:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4286 00000100:00080000:2.0:1788879299.709836:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879299.709845:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879299.709854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879299.709855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879299.709885:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511606784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879299.709904:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879299.709909:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879299/real 1788879299] req@ffffa061888ace00 x1875771511606528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879299.709913:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879299.709921:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879299.709938:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4286 00000100:00100000:0.0:1788879299.709946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511606528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879299.709947:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879299.709956:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879299.709972:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879299.709979:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879299.709987:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879299.709993:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511606528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879299.710002:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511606912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879299.710027:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879299.710028:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606656:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788879299.710042:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879299.710048:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879299.710057:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4286 00000800:00000100:0.0:1788879299.710066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879299.710067:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879299.710101:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879299.710112:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879299.710120:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879299/real 1788879299] req@ffffa061888ad180 x1875771511606656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879299.710141:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511607040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879299.710142:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511606656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879299.710174:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879299.710186:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879299.710193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879299.710202:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879299.710227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879299.710239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879299/real 1788879299] req@ffffa061888ac700 x1875771511606784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879299.710252:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511606784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879299.710269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879299.710272:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879299.710276:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879299.710291:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879299.710308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879299.710319:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879299/real 1788879299] req@ffffa061888aed80 x1875771511606912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879299.710331:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511606912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879299.710341:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879299.710344:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879299.710347:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511606912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879299.710354:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879299.710369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879299.710378:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879299/real 1788879299] req@ffffa061888ad880 x1875771511607040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879299.710418:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511607040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879299.710427:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879299.710431:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879299.710435:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879300.349481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879300.349668:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879300.605482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879301.373474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879301.373545:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879301.629465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879302.397513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879302.397625:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879302.653490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879303.421498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879303.421654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879303.677493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879304.445454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879304.445574:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879304.701451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879304.829481:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879304.829489:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879304.829493:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879304.829501:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4291 00000100:00080000:3.0:1788879304.829512:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879304.829518:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879304.829553:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511607168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879304.829576:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879304.829579:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879304.829581:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879304.829584:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4291 00000100:00080000:3.0:1788879304.829587:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879304.829590:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879304.829598:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879304.829608:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511607296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879304.829614:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511607168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879304.829619:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879304.829622:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879304.829624:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879304.829627:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4291 00000100:00080000:3.0:1788879304.829629:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879304.829631:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879304.829634:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879304.829642:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511607424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879304.829648:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879304.829651:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879304.829652:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879304.829654:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4291 00000100:00080000:3.0:1788879304.829657:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879304.829661:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879304.829665:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879304.829671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511607552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879304.829677:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879304.829679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879304.829681:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879304.829683:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4291 00000100:00080000:3.0:1788879304.829685:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879304.829687:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879304.829690:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879304.829699:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511607680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879304.829728:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879304.829757:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879304.829781:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879304.829795:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879305.469478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879305.469549:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879305.725486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879306.493539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879306.493708:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879306.749531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879307.517533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879307.517673:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879307.773459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879307.901722:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879307.901784:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879307.901809:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879307.901859:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879307.901875:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879307.901893:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879307.901911:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879307.901935:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879304/real 1788879307] req@ffffa06188870700 x1875771511607296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879307.901981:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511607296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879307.902002:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879307.902010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879307.902045:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879307.902060:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879304/real 1788879307] req@ffffa06188873b80 x1875771511607424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879307.902084:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511607424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879307.902099:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879307.902105:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879307.902111:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879307.902123:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879304/real 1788879307] req@ffffa06188872300 x1875771511607552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879307.902146:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511607552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879307.902160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879307.902166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879307.902172:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879307.902184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879304/real 1788879307] req@ffffa06188872a00 x1875771511607680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879307.902205:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511607680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879307.902219:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879307.902225:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879307.902231:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879307.902242:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879304/real 1788879307] req@ffffa06188872d80 x1875771511607168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879307.902265:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511607168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879307.902279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879307.902285:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879307.902291:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511607168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879308.541497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879308.541593:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879308.797526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879309.565562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879309.565631:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879309.821516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879309.949594:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879309.949608:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879309.949613:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879309.949629:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4296 00000100:00080000:3.0:1788879309.949647:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879309.949662:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879309.949717:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511607808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879309.949758:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879309.949766:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879309.949769:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879309.949774:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4296 00000100:00080000:3.0:1788879309.949781:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879309.949788:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879309.949802:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879309.949825:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511607936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879309.949829:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511607808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879309.949846:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879309.949873:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879309.949878:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879309.949884:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4296 00000100:00080000:3.0:1788879309.949889:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879309.949893:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879309.949900:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879309.949916:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511608064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879309.949945:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879309.949951:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879309.949953:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879309.949958:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4296 00000100:00080000:3.0:1788879309.949963:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879309.950002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879309.950029:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879309.950046:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879309.950073:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511608192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879309.950091:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879309.950099:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879309/real 1788879309] req@ffffa06188873480 x1875771511607808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879309.950100:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879309.950108:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879309.950133:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4296 00000100:00080000:3.0:1788879309.950147:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879309.950159:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879309.950170:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879309.950172:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511607808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879309.950202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788879309.950204:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511608320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879309.950218:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879309.950238:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511607808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879309.950289:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879309.950338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879309.950364:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879309/real 1788879309] req@ffffa06188871500 x1875771511607936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879309.950428:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511607936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879309.950447:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879309.950453:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879309.950459:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511607936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879309.950471:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879309.950511:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879309.950533:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879309/real 1788879309] req@ffffa06188873100 x1875771511608064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879309.950556:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511608064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879309.950571:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879309.950577:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879309.950584:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879309.950596:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879309.950635:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879309.950656:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879309/real 1788879309] req@ffffa06188872680 x1875771511608192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879309.950676:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511608192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879309.950691:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879309.950696:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879309.950703:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879309.950715:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879309.950752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879309.950772:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879309/real 1788879309] req@ffffa06188872d80 x1875771511608320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879309.950792:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511608320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879309.950807:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879309.950812:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879309.950818:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879310.589529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879310.589734:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879310.845587:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879311.613479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879311.613721:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879311.869510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879312.637533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879312.637707:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879312.893483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879313.661511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879313.661603:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879313.917517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879314.685479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879314.685619:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879314.941473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879315.069549:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879315.069565:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879315.069572:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879315.069589:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4301 00000100:00080000:3.0:1788879315.069607:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879315.069618:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879315.069681:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511608448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879315.069740:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879315.069745:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879315.069748:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879315.069753:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4301 00000100:00080000:3.0:1788879315.069759:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879315.069766:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879315.069780:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879315.069800:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511608576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879315.069810:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511608448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879315.069820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879315.069829:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879315.069832:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879315.069838:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4301 00000100:00080000:3.0:1788879315.069842:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879315.069863:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879315.069870:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879315.069887:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511608704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879315.069904:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879315.069909:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879315.069913:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879315.069920:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4301 00000100:00080000:3.0:1788879315.069927:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879315.069932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879315.069942:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879315.069966:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879315.069995:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511608832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879315.070007:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879315/real 1788879315] req@ffffa06188872a00 x1875771511608448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879315.070034:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879315.070046:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879315.070050:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879315.070057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511608448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879315.070059:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4301 00000100:00080000:3.0:1788879315.070074:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879315.070079:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879315.070086:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879315.070089:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879315.070099:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511608448:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879315.070101:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879315.070124:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511608960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879315.070135:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879315.070170:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879315.070185:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879315/real 1788879315] req@ffffa06188872300 x1875771511608576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879315.070198:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511608576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879315.070208:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879315.070212:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879315.070215:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879315.070222:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879315.070240:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879315.070249:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879315/real 1788879315] req@ffffa06188873b80 x1875771511608704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879315.070261:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511608704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879315.070269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879315.070272:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879315.070275:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879315.070281:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879315.070298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879315.070308:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879315/real 1788879315] req@ffffa06188870700 x1875771511608832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879315.070320:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511608832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879315.070328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879315.070331:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879315.070334:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879315.070364:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879315.070379:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879315.070391:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879315/real 1788879315] req@ffffa06188873800 x1875771511608960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879315.070422:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511608960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879315.070470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879315.070477:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879315.070481:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511608960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879315.709457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879315.709584:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879315.965519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879316.733469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879316.733731:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879316.989483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879317.757500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879317.757582:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879318.013452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879318.781493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879318.781686:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879319.037502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879319.805516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879319.805591:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879320.061506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879320.189554:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879320.189576:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879320.189582:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879320.189598:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4306 00000100:00080000:2.0:1788879320.189614:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879320.189625:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879320.189676:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511609088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879320.189715:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879320.189721:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879320.189724:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879320.189731:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4306 00000100:00080000:2.0:1788879320.189750:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879320.189758:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879320.189773:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879320.189787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511609088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879320.189798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511609216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879320.189819:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879320.189824:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879320.189828:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879320.189834:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4306 00000100:00080000:2.0:1788879320.189840:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879320.189847:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879320.189855:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879320.189873:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511609344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879320.189889:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879320.189897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879320.189900:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879320.189907:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879320.189915:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4306 00000100:00080000:2.0:1788879320.189953:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879320.189962:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879320.189972:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879320.189987:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879320/real 1788879320] req@ffffa061888ac380 x1875771511609088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879320.189998:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511609472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879320.190020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511609088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879320.190043:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879320.190047:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879320.190050:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879320.190060:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511609088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879320.190062:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879320.190071:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879320.190081:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4306 00000100:00080000:2.0:1788879320.190088:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879320.190096:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879320.190100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609216:192.168.203.111@tcp:38:kworker.0 00000008:00100000:2.0:1788879320.190111:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879320.190141:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511609600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879320.190142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879320.190159:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879320/real 1788879320] req@ffffa061888ae680 x1875771511609216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879320.190200:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511609216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879320.190217:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879320.190224:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879320.190241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879320.190255:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879320.190285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879320.190309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879320/real 1788879320] req@ffffa061888adc00 x1875771511609344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879320.190337:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511609344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879320.190350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879320.190355:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879320.190371:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879320.190423:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879320.190453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879320.190482:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879320/real 1788879320] req@ffffa061888afb80 x1875771511609472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879320.190495:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511609472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879320.190504:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879320.190507:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879320.190511:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879320.190530:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879320.190547:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879320.190557:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879320/real 1788879320] req@ffffa061888aea00 x1875771511609600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879320.190568:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511609600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879320.190583:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879320.190587:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879320.190590:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879320.829550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879320.829677:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879321.085477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879321.853464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879321.853559:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879322.109472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879322.877467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879322.877561:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879323.133485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879323.901528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879323.901678:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879324.157502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879324.925543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879324.925628:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879325.181486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879325.309488:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879325.309499:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879325.309504:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879325.309517:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4312 00000100:00080000:2.0:1788879325.309529:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879325.309539:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879325.309582:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511609728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879325.309611:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879325.309614:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879325.309616:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879325.309621:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4312 00000100:00080000:2.0:1788879325.309625:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879325.309637:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879325.309650:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879325.309652:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511609728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879325.309667:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511609856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879325.309679:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879325.309682:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879325.309684:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879325.309688:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4312 00000100:00080000:2.0:1788879325.309691:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879325.309695:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879325.309699:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879325.309717:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511609984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879325.309726:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879325.309730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879325.309735:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879325.309736:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879325.309745:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4312 00000100:00080000:2.0:1788879325.309751:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879325.309755:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879325.309761:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879325.309774:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511610112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879325.309784:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879325/real 1788879325] req@ffffa061888ae300 x1875771511609728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879325.309786:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879325.309799:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879325.309804:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879325.309810:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4312 00000100:00080000:2.0:1788879325.309814:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879325.309818:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879325.309821:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511609728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879325.309824:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879325.309841:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788879325.309843:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511610240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879325.309851:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879325.309861:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511609728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879325.309916:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879325.309962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879325.309979:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879325/real 1788879325] req@ffffa061888af100 x1875771511609856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879325.309995:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511609856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879325.310006:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879325.310010:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879325.310014:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879325.310021:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879325.310040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879325.310051:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879325/real 1788879325] req@ffffa061888ad500 x1875771511609984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879325.310064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511609984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879325.310073:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879325.310076:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879325.310080:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511609984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879325.310086:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879325.310104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879325.310115:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879325/real 1788879325] req@ffffa061888ac000 x1875771511610112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879325.310128:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511610112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879325.310136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879325.310139:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879325.310144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879325.310150:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879325.310167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879325.310176:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879325/real 1788879325] req@ffffa061888adf80 x1875771511610240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879325.310189:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511610240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879325.310198:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879325.310201:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879325.310204:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879325.949568:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879325.949622:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879326.205518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879326.973480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879326.973594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879327.229466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879327.997504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879327.997610:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879328.253536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879329.021493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879329.021637:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879329.277506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879330.045526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879330.045709:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879330.301559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879330.429553:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879330.429568:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879330.429574:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879330.429590:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4317 00000100:00080000:2.0:1788879330.429606:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879330.429619:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879330.429672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511610368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879330.429710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879330.429715:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879330.429717:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879330.429723:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4317 00000100:00080000:2.0:1788879330.429727:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879330.429731:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879330.429745:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879330.429764:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511610496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879330.429797:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511610368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879330.429804:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879330.429812:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879330.429815:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879330.429821:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4317 00000100:00080000:2.0:1788879330.429825:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879330.429829:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879330.429835:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879330.429853:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511610624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879330.429893:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879330.429899:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879330.429901:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879330.429906:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4317 00000100:00080000:2.0:1788879330.429912:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879330.429917:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879330.429924:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879330.429933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879330.429944:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511610752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879330.429960:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879330.429965:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879330.429967:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879330.429972:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4317 00000100:00080000:2.0:1788879330.429978:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879330.429984:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879330.429990:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879330.429998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879330/real 1788879330] req@ffffa061888af480 x1875771511610368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879330.430011:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511610880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879330.430044:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511610368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879330.430066:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879330.430073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879330.430081:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511610368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879330.430116:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879330.430147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879330.430163:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879330/real 1788879330] req@ffffa061888aca80 x1875771511610496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879330.430179:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511610496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879330.430190:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879330.430195:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879330.430199:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879330.430207:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879330.430227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879330.430239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879330/real 1788879330] req@ffffa061888adf80 x1875771511610624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879330.430253:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511610624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879330.430269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879330.430276:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879330.430283:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879330.430295:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879330.430334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879330.430358:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879330/real 1788879330] req@ffffa061888ac000 x1875771511610752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879330.430412:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511610752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879330.430425:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879330.430429:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879330.430433:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879330.430441:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879330.430471:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879330.430487:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879330/real 1788879330] req@ffffa061888ad500 x1875771511610880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879330.430501:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511610880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879330.430512:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879330.430516:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879330.430519:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511610880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879331.069591:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879331.069660:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879331.325562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879332.093529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879332.093681:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879332.349504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879333.117479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879333.117593:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879333.373489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879334.141456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879334.141552:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879334.397469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788879335.165490:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879335.165609:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788879335.421456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879335.549472:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879335.549482:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879335.549487:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879335.549498:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4322 00000100:00080000:2.0:1788879335.549509:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879335.549518:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879335.549576:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511611008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879335.549598:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879335.549603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879335.549605:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879335.549610:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4322 00000100:00080000:2.0:1788879335.549614:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879335.549619:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879335.549629:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879335.549637:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511611008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879335.549642:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511611136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879335.549652:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879335.549655:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879335.549657:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879335.549661:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4322 00000100:00080000:2.0:1788879335.549664:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879335.549667:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879335.549672:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879335.549681:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511611264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879335.549689:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879335.549692:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879335.549694:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879335.549697:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4322 00000100:00080000:2.0:1788879335.549700:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879335.549703:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879335.549707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788879335.549708:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879335.549732:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511611392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879335.549740:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879335.549744:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879335.549746:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879335.549750:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4322 00000100:00080000:2.0:1788879335.549753:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879335.549759:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879335.549760:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879335/real 1788879335] req@ffffa061888af100 x1875771511611008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879335.549766:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879335.549777:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511611520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879335.549783:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511611008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879335.549796:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879335.549802:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879335.549806:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511611008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879335.549847:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879335.549892:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879335.549902:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879335/real 1788879335] req@ffffa061888ae300 x1875771511611136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879335.549910:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511611136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879335.549928:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879335.549930:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879335.549933:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879335.549937:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879335.549948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879335.549954:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879335/real 1788879335] req@ffffa061888aea00 x1875771511611264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879335.549962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511611264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879335.549972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879335.549974:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879335.549976:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879335.549979:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879335.550000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879335.550006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879335/real 1788879335] req@ffffa061888afb80 x1875771511611392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879335.550013:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511611392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879335.550019:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879335.550021:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879335.550022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879335.550026:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879335.550043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879335.550050:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879335/real 1788879335] req@ffffa061888adc00 x1875771511611520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879335.550057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511611520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879335.550062:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879335.550064:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879335.550067:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879336.189455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879336.189542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879336.445451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879337.213526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879337.213687:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879337.469537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879338.237573:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879338.237630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879338.493520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879339.261534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879339.261632:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879339.517557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879340.285470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879340.285720:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879340.541538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879340.669454:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879340.669463:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879340.669467:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879340.669477:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4327 00000100:00080000:2.0:1788879340.669489:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879340.669498:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879340.669536:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511611648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879340.669558:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879340.669561:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879340.669563:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879340.669566:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4327 00000100:00080000:2.0:1788879340.669568:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879340.669571:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879340.669581:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879340.669591:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511611776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879340.669595:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511611648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879340.669614:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879340.669618:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879340.669620:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879340.669625:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4327 00000100:00080000:2.0:1788879340.669629:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879340.669634:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879340.669640:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879340.669654:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511611904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879340.669664:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879340.669670:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879340.669671:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879340.669675:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879340.669679:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4327 00000100:00080000:2.0:1788879340.669683:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879340.669687:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879340.669692:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879340.669702:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511612032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879340.669708:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879340/real 1788879340] req@ffffa061888ae680 x1875771511611648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879340.669714:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879340.669722:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879340.669726:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879340.669729:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4327 00000100:00080000:2.0:1788879340.669732:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879340.669736:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879340.669738:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511611648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879340.669740:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879340.669752:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511612160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879340.669753:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879340.669762:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879340.669768:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511611648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879340.669788:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879340.669820:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879340.669829:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879340/real 1788879340] req@ffffa061888ac380 x1875771511611776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879340.669838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511611776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879340.669844:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879340.669847:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879340.669849:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879340.669853:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879340.669863:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879340.669886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879340/real 1788879340] req@ffffa061888ad880 x1875771511611904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879340.669894:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511611904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879340.669900:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879340.669901:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879340.669903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511611904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879340.669907:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879340.669919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879340.669925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879340/real 1788879340] req@ffffa061888aed80 x1875771511612032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879340.669933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511612032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879340.669939:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879340.669940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879340.669942:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879340.669946:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879340.669956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879340.669961:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879340/real 1788879340] req@ffffa061888ac700 x1875771511612160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879340.669968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511612160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879340.669974:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879340.669976:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879340.669978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879341.309479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879341.309548:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879341.565454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879342.333542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879342.333651:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879342.589563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879343.357491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879343.357698:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879343.613483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879344.381498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879344.381618:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879344.637518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879345.405514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879345.405661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879345.661449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879345.789485:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879345.789505:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879345.789509:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879345.789519:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4332 00000100:00080000:2.0:1788879345.789531:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879345.789539:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879345.789584:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511612288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879345.789608:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879345.789611:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879345.789613:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879345.789617:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4332 00000100:00080000:2.0:1788879345.789619:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879345.789623:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879345.789635:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879345.789656:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511612288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879345.789657:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511612416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879345.789680:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879345.789684:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879345.789687:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879345.789692:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4332 00000100:00080000:2.0:1788879345.789698:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879345.789703:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879345.789710:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879345.789740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511612544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879345.789748:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879345.789750:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879345.789752:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879345.789754:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4332 00000800:00000100:0.0:1788879345.789772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879345.789773:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879345.789780:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879345.789785:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879345.789817:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511612672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879345.789830:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879345.789834:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879345.789837:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879345.789842:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879345/real 1788879345] req@ffffa061888ad180 x1875771511612288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879345.789847:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4332 00000100:00080000:2.0:1788879345.789856:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879345.789866:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879345.789882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511612288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879345.789883:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879345.789900:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788879345.789902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511612800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879345.789909:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879345.789919:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511612288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879345.789964:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879345.789994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879345.790004:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879345/real 1788879345] req@ffffa061888ace00 x1875771511612416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879345.790031:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511612416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879345.790039:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879345.790042:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879345.790044:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879345.790050:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879345.790065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879345.790073:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879345/real 1788879345] req@ffffa061888ac700 x1875771511612544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879345.790082:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511612544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879345.790088:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879345.790090:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879345.790093:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879345.790097:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879345.790110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879345.790117:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879345/real 1788879345] req@ffffa061888aed80 x1875771511612672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879345.790126:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511612672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879345.790132:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879345.790135:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879345.790137:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879345.790141:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879345.790152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879345.790159:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879345/real 1788879345] req@ffffa061888ad880 x1875771511612800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879345.790169:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511612800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879345.790175:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879345.790177:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879345.790180:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511612800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879346.429506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879346.429740:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879346.685524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879347.453532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879347.453576:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879347.709475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879348.477464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879348.477544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879348.733462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879349.501483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879349.501555:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879349.757465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879350.525509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879350.525662:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879350.781511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879350.909501:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879350.909512:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879350.909518:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879350.909532:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4337 00000100:00080000:2.0:1788879350.909545:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879350.909554:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879350.909599:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511612928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879350.909628:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879350.909632:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879350.909634:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879350.909639:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4337 00000100:00080000:2.0:1788879350.909642:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879350.909647:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879350.909661:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879350.909682:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511612928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879350.909682:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511613056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879350.909704:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879350.909707:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879350.909709:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879350.909712:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4337 00000100:00080000:2.0:1788879350.909715:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879350.909719:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879350.909740:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879350.909757:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511613184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879350.909767:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879350.909772:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879350.909774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879350.909778:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4337 00000100:00080000:2.0:1788879350.909782:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879350.909790:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879350.909791:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788879350.909798:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879350.909813:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511613312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879350.909823:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879350.909827:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879350.909829:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879350.909834:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4337 00000100:00100000:0.0:1788879350.909839:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879350/real 1788879350] req@ffffa061888ac380 x1875771511612928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879350.909848:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879350.909858:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879350.909865:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879350.909872:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511612928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879350.909879:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511613440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879350.909901:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879350.909909:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879350.909916:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511612928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879350.909944:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879350.909966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879350.909978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879350/real 1788879350] req@ffffa061888ae680 x1875771511613056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879350.909990:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511613056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879350.910005:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879350.910008:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879350.910012:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879350.910017:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879350.910031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879350.910040:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879350/real 1788879350] req@ffffa061888adc00 x1875771511613184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879350.910051:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511613184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879350.910058:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879350.910061:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879350.910064:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879350.910070:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879350.910084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879350.910091:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879350/real 1788879350] req@ffffa061888afb80 x1875771511613312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879350.910102:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511613312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879350.910109:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879350.910112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879350.910115:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879350.910121:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879350.910134:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879350.910142:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879350/real 1788879350] req@ffffa061888aea00 x1875771511613440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879350.910152:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511613440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879350.910160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879350.910162:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879350.910165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879351.549545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879351.549682:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879351.805500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879352.573476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879352.573574:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879352.829508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879353.597472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879353.597528:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879353.853493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879354.621487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879354.621573:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879354.877489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879355.645449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879355.645567:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879355.901488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879356.029480:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879356.029489:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879356.029493:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879356.029504:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4342 00000100:00080000:2.0:1788879356.029514:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879356.029521:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879356.029557:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511613568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879356.029579:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879356.029582:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879356.029583:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879356.029586:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4342 00000100:00080000:2.0:1788879356.029589:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879356.029592:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879356.029602:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879356.029612:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511613696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879356.029617:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511613568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879356.029624:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879356.029627:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879356.029628:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879356.029631:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4342 00000100:00080000:2.0:1788879356.029632:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879356.029636:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879356.029640:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879356.029647:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511613824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879356.029653:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879356.029655:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879356.029656:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879356.029659:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4342 00000100:00080000:2.0:1788879356.029661:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879356.029664:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879356.029667:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879356.029676:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511613952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879356.029684:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879356.029703:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879356.029708:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879356.029711:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879356.029716:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4342 00000100:00080000:2.0:1788879356.029728:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879356.029733:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879356.029736:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879356/real 1788879356] req@ffffa061888ae300 x1875771511613568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879356.029740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879356.029765:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511613568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879356.029765:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511614080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879356.029786:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879356.029791:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879356.029796:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511613568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879356.029816:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879356.029833:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879356.029842:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879356/real 1788879356] req@ffffa061888af100 x1875771511613696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879356.029850:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511613696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879356.029857:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879356.029859:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879356.029862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879356.029865:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879356.029875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879356.029881:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879356/real 1788879356] req@ffffa061888ad500 x1875771511613824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879356.029889:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511613824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879356.029894:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879356.029896:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879356.029898:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879356.029902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879356.029912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879356.029917:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879356/real 1788879356] req@ffffa061888ac000 x1875771511613952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879356.029925:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511613952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879356.029930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879356.029932:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879356.029934:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511613952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879356.029938:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879356.029946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879356.029951:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879356/real 1788879356] req@ffffa061888adf80 x1875771511614080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879356.029960:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511614080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879356.029968:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879356.029971:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879356.029974:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879356.669445:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879356.669526:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879356.925508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879357.693449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879357.693546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879357.949456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879358.717456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879358.717567:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879358.973453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879359.741461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879359.741554:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879359.997460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879360.765497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879360.765588:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879361.021493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879361.149544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879361.149563:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879361.149571:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879361.149600:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4347 00000100:00080000:2.0:1788879361.149618:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879361.149632:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879361.149737:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511614208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879361.149767:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879361.149770:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879361.149772:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879361.149777:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4347 00000100:00080000:2.0:1788879361.149783:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879361.149789:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879361.149799:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879361.149815:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511614208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879361.149818:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511614336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879361.149842:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879361.149847:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879361.149850:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879361.149857:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4347 00000100:00080000:2.0:1788879361.149863:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879361.149870:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879361.149878:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879361.149897:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511614464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879361.149907:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879361.149911:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879361.149913:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879361.149917:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4347 00000100:00080000:2.0:1788879361.149922:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879361.149926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879361.149933:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879361.149942:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879361.149959:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511614592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879361.149984:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879361.149989:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879361.149992:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879361.149999:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879361/real 1788879361] req@ffffa061888aca80 x1875771511614208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879361.150006:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4347 00000100:00080000:2.0:1788879361.150017:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879361.150029:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879361.150039:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879361.150042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511614208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879361.150060:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788879361.150063:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511614720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879361.150079:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879361.150095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511614208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879361.150148:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879361.150181:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879361.150197:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879361/real 1788879361] req@ffffa061888af480 x1875771511614336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879361.150211:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511614336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879361.150221:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879361.150225:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879361.150228:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879361.150236:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879361.150260:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879361.150270:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879361/real 1788879361] req@ffffa061888adf80 x1875771511614464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879361.150281:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511614464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879361.150291:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879361.150295:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879361.150299:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879361.150307:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879361.150332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879361.150359:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879361/real 1788879361] req@ffffa061888ac000 x1875771511614592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879361.150374:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511614592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879361.150428:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879361.150432:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879361.150436:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879361.150442:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879361.150462:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879361.150471:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879361/real 1788879361] req@ffffa061888ad500 x1875771511614720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879361.150480:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511614720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879361.150486:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879361.150489:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879361.150491:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879361.789484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879361.789625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879362.045487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879362.813551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879362.813602:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879363.069499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879363.837567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879363.837757:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879364.093626:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879364.861480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879364.861539:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879365.118421:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879365.885502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879365.885717:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879366.141681:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879366.269465:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879366.269476:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879366.269482:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879366.269495:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4352 00000100:00080000:2.0:1788879366.269509:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879366.269518:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879366.269562:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511614848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879366.269591:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879366.269595:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879366.269598:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879366.269601:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4352 00000100:00080000:2.0:1788879366.269606:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879366.269610:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879366.269644:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879366.269665:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511614848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879366.269674:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511614976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879366.269687:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879366.269690:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879366.269692:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879366.269704:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4352 00000100:00080000:2.0:1788879366.269708:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879366.269724:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879366.269729:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879366.269741:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511615104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879366.269750:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879366.269754:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879366.269755:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879366.269758:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4352 00000100:00080000:2.0:1788879366.269762:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879366.269767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879366.269771:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879366.269778:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879366.269791:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511615232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879366.269799:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879366.269803:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879366.269805:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879366.269810:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4352 00000100:00080000:2.0:1788879366.269813:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879366.269816:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879366/real 1788879366] req@ffffa061888af100 x1875771511614848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879366.269821:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879366.269830:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879366.269847:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511615360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879366.269855:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511614848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879366.269878:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879366.269884:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879366.269889:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511614848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879366.269914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879366.269934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879366.269945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879366/real 1788879366] req@ffffa061888ae300 x1875771511614976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879366.269956:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511614976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879366.269964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879366.269967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879366.269970:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511614976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879366.269975:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879366.269988:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879366.269996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879366/real 1788879366] req@ffffa061888aea00 x1875771511615104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879366.270006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511615104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879366.270013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879366.270016:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879366.270018:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879366.270024:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879366.270038:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879366.270046:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879366/real 1788879366] req@ffffa061888afb80 x1875771511615232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879366.270056:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511615232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879366.270063:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879366.270065:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879366.270068:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879366.270072:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879366.270084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879366.270092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879366/real 1788879366] req@ffffa061888adc00 x1875771511615360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879366.270102:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511615360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879366.270108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879366.270111:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879366.270113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879366.909504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879366.909589:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879367.165445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879367.933559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879367.933727:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879368.189535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879368.957523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879368.957706:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879369.213504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879369.981469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879369.981674:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879370.237493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879371.005518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879371.005704:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879371.261540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879371.389512:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879371.389523:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879371.389527:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879371.389539:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4358 00000100:00080000:2.0:1788879371.389557:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879371.389566:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879371.389620:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511615488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879371.389645:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879371.389649:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879371.389651:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879371.389655:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4358 00000100:00080000:2.0:1788879371.389659:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879371.389664:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879371.389676:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879371.389686:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511615488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879371.389696:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511615616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879371.389713:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879371.389728:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879371.389731:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879371.389735:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4358 00000100:00080000:2.0:1788879371.389739:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879371.389744:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879371.389750:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879371.389762:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511615744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879371.389771:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879371.389775:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879371.389777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879371.389781:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4358 00000100:00080000:2.0:1788879371.389785:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879371.389787:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879371.389793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879371.389822:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879371.389832:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511615872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879371.389861:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879371.389866:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879371.389869:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879371.389874:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4358 00000100:00080000:2.0:1788879371.389881:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879371.389886:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879371.389893:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879371.389918:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879371.389925:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511616000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879371.389987:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879371.390014:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879372.029594:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879372.029700:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879372.285540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879373.053522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879373.053695:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879373.309659:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879374.077555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879374.077663:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879374.333496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788879374.461908:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788879374.461962:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788879374.461983:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879374.462043:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879374.462053:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879374.462075:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879374.462089:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879374.462146:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879371/real 1788879374] req@ffffa061888aed80 x1875771511615872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879374.462204:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511615872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879374.462227:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879374.462237:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879374.462255:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879374.462270:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879371/real 1788879374] req@ffffa061888ac700 x1875771511616000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879374.462292:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511616000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879374.462308:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879374.462313:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879374.462319:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879374.462332:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879371/real 1788879374] req@ffffa061888ac380 x1875771511615616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879374.462357:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511615616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879374.462374:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879374.462416:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879374.462425:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879374.462441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879371/real 1788879374] req@ffffa061888ad880 x1875771511615744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879374.462468:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511615744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879374.462485:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879374.462494:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879374.462500:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511615744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879374.462515:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879371/real 1788879374] req@ffffa061888ae680 x1875771511615488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879374.462540:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511615488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879374.462555:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879374.462582:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879374.462589:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511615488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879375.101536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879375.101679:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879375.357506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879376.125493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879376.125754:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879376.381522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879376.509529:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879376.509544:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879376.509574:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879376.509592:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4363 00000100:00080000:2.0:1788879376.509609:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879376.509621:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879376.509683:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511616128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879376.509753:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879376.509760:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879376.509764:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879376.509773:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4363 00000100:00080000:2.0:1788879376.509779:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879376.509785:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879376.509800:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879376.509822:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511616256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879376.509839:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879376.509843:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879376.509848:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879376.509854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511616128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879376.509856:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4363 00000100:00080000:2.0:1788879376.509865:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879376.509879:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879376.509890:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879376.509918:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511616384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879376.509938:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879376.509945:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879376.509948:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879376.509956:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4363 00000100:00080000:2.0:1788879376.509966:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879376.509980:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879376.509983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788879376.509993:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879376.510034:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511616512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879376.510057:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879376/real 1788879376] req@ffffa061888ace00 x1875771511616128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879376.510058:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879376.510075:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879376.510082:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879376.510116:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4363 00000100:00080000:2.0:1788879376.510124:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879376.510137:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879376.510149:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879376.510174:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511616128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879376.510187:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511616640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879376.510209:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879376.510234:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879376.510244:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511616128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879376.510301:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879376.510347:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879376.510372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879376/real 1788879376] req@ffffa061888ad180 x1875771511616256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879376.510438:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511616256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879376.510455:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879376.510461:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879376.510468:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879376.510481:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879376.510520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879376.510542:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879376/real 1788879376] req@ffffa061888ae680 x1875771511616384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879376.510593:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511616384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879376.510621:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879376.510627:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879376.510634:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879376.510646:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879376.510695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879376.510717:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879376/real 1788879376] req@ffffa061888ad880 x1875771511616512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879376.510741:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511616512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879376.510755:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879376.510762:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879376.510767:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879376.510778:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879376.510815:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879376.510835:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879376/real 1788879376] req@ffffa061888ac380 x1875771511616640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879376.510854:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511616640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879376.510868:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879376.510875:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879376.510884:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879377.149463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879377.149553:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879377.405519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879378.173460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879378.173729:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879378.429469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879379.197461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879379.197513:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879379.453515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879380.221602:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879380.221712:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879380.477574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879381.245483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879381.245578:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879381.501583:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879381.629573:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879381.629585:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879381.629592:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879381.629607:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4368 00000100:00080000:2.0:1788879381.629621:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879381.629631:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879381.629682:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511616768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879381.629716:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879381.629721:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879381.629723:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879381.629727:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4368 00000100:00080000:2.0:1788879381.629731:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879381.629735:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879381.629756:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879381.629770:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511616896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879381.629782:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879381.629786:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879381.629788:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879381.629792:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4368 00000100:00080000:2.0:1788879381.629795:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879381.629799:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879381.629804:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879381.629813:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511616768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879381.629818:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511617024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879381.629835:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879381.629841:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879381.629845:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879381.629851:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4368 00000100:00080000:2.0:1788879381.629856:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879381.629863:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879381.629872:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879381.629891:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511617152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879381.629907:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879381.629912:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879381.629915:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879381.629921:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4368 00000800:00000100:0.0:1788879381.629927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879381.629931:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879381.629940:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879381.629947:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879381.629963:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511617280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879381.630000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879381/real 1788879381] req@ffffa061888ac700 x1875771511616768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879381.630030:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511616768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879381.630048:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879381.630055:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879381.630061:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511616768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879381.630071:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879381.630106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879381.630126:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879381/real 1788879381] req@ffffa061888aed80 x1875771511616896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879381.630146:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511616896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879381.630159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879381.630165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879381.630170:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511616896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879381.630211:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879381.630236:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879381.630249:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879381/real 1788879381] req@ffffa061888adc00 x1875771511617024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879381.630261:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511617024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879381.630269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879381.630272:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879381.630276:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879381.630281:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879381.630297:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879381.630306:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879381/real 1788879381] req@ffffa061888afb80 x1875771511617152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879381.630316:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511617152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879381.630324:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879381.630326:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879381.630329:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879381.630334:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879381.630351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879381.630365:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879381/real 1788879381] req@ffffa061888aea00 x1875771511617280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879381.630419:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511617280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879381.630443:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879381.630448:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879381.630452:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879382.269644:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879382.269697:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879382.525638:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788879383.293633:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879383.293645:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788879383.549648:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788879384.317588:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879384.317599:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788879384.573631:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879385.341591:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879385.341660:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879385.597559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879386.365638:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879386.365715:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879386.621627:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879386.749638:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879386.749652:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879386.749659:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879386.749675:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4373 00000100:00080000:2.0:1788879386.749696:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879386.749709:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879386.749787:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511617408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879386.749836:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879386.749842:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879386.749845:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879386.749854:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4373 00000100:00080000:2.0:1788879386.749861:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879386.749869:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879386.749884:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879386.749907:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511617536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879386.749908:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511617408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879386.749946:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879386.749952:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879386.749956:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879386.749962:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4373 00000100:00080000:2.0:1788879386.749967:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879386.749972:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879386.749990:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879386.750005:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511617664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879386.750014:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879386.750018:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879386.750019:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879386.750024:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4373 00000100:00080000:2.0:1788879386.750028:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879386.750031:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879386.750036:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879386.750060:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511617792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879386.750068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879386.750072:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879386.750077:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879386.750083:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879386.750091:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4373 00000100:00080000:2.0:1788879386.750095:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879386.750101:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879386.750105:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879386.750126:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511617920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879386.750167:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879386/real 1788879386] req@ffffa061888ae300 x1875771511617408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879386.750195:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511617408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879386.750209:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879386.750215:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879386.750220:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511617408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879386.750251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879386.750280:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879386.750295:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879386/real 1788879386] req@ffffa061888af100 x1875771511617536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879386.750309:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511617536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879386.750319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879386.750323:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879386.750327:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879386.750335:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879386.750354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879386.750365:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879386/real 1788879386] req@ffffa061888ad500 x1875771511617664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879386.750378:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511617664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879386.750387:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879386.750390:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879386.750393:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879386.750433:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879386.750472:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879386.750507:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879386/real 1788879386] req@ffffa061888ac000 x1875771511617792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879386.750519:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511617792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879386.750527:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879386.750530:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879386.750551:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879386.750559:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879386.750583:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879386.750595:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879386/real 1788879386] req@ffffa061888adf80 x1875771511617920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879386.750609:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511617920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879386.750618:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879386.750621:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879386.750625:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511617920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879387.389572:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879387.389578:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879387.645594:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879388.413530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879388.413646:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879388.669530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879389.437540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879389.437575:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879389.693547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879390.461576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879390.461679:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879390.717507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879391.485537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879391.485577:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879391.741486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879391.869504:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879391.869513:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879391.869517:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879391.869528:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4378 00000100:00080000:1.0:1788879391.869538:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879391.869545:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879391.869583:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511618048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879391.869605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879391.869607:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879391.869609:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879391.869612:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4378 00000100:00080000:1.0:1788879391.869614:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879391.869619:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879391.869628:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879391.869637:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511618176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879391.869643:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511618048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879391.869649:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879391.869653:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879391.869654:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879391.869656:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4378 00000100:00080000:1.0:1788879391.869658:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879391.869660:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879391.869663:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879391.869672:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511618304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879391.869678:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879391.869681:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879391.869682:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879391.869685:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4378 00000100:00080000:1.0:1788879391.869705:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879391.869708:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879391.869712:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879391.869720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879391.869721:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511618432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879391.869730:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879391.869733:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879391.869735:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879391.869737:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4378 00000100:00080000:1.0:1788879391.869741:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879391.869743:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879391.869746:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879391.869753:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511618560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879391.869767:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879391/real 1788879391] req@ffffa06188698380 x1875771511618048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879391.869786:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511618048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879391.869795:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879391.869798:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879391.869801:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511618048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879391.869824:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879391.869842:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879391.869850:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879391/real 1788879391] req@ffffa0618869b800 x1875771511618176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879391.869858:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511618176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879391.869868:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879391.869870:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879391.869873:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879391.869876:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879391.869887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879391.869892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879391/real 1788879391] req@ffffa06188699c00 x1875771511618304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879391.869899:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511618304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879391.869905:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879391.869906:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879391.869908:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879391.869912:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879391.869922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879391.869928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879391/real 1788879391] req@ffffa0618869aa00 x1875771511618432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879391.869935:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511618432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879391.869940:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879391.869942:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879391.869944:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879391.869948:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879391.869958:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879391.869963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879391/real 1788879391] req@ffffa0618869b100 x1875771511618560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879391.869970:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511618560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879391.869976:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879391.869977:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879391.869979:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879392.509551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879392.509582:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879392.765574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879393.533485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879393.533549:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879393.789472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879394.557476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879394.557530:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879394.813462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879395.581554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879395.581702:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879395.837477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879396.605608:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879396.605682:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879396.861589:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879396.989695:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879396.989710:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879396.989717:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879396.989732:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4383 00000100:00080000:1.0:1788879396.989748:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879396.989759:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879396.989819:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511618688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879396.989850:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879396.989855:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879396.989857:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879396.989863:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4383 00000100:00080000:1.0:1788879396.989867:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879396.989872:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879396.989884:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879396.989900:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511618816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879396.989913:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879396.989921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511618688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879396.989921:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879396.989943:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879396.989957:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4383 00000100:00080000:1.0:1788879396.989962:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879396.989966:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879396.989973:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879396.989991:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511618944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879396.990021:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879396.990026:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879396.990029:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879396.990034:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4383 00000100:00080000:1.0:1788879396.990050:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879396.990055:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879396.990063:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879396.990070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879396.990084:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511619072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879396.990121:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879396.990124:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879396.990127:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879396.990132:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4383 00000100:00080000:1.0:1788879396.990138:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879396.990143:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879396.990149:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879396.990152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879396/real 1788879396] req@ffffa0618869b100 x1875771511618688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879396.990164:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511619200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879396.990215:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511618688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879396.990235:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879396.990240:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879396.990245:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511618688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879396.990296:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879396.990346:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879396.990365:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879396/real 1788879396] req@ffffa0618869aa00 x1875771511618816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879396.990448:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511618816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879396.990464:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879396.990471:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879396.990477:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879396.990487:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879396.990548:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879396.990591:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879396/real 1788879396] req@ffffa06188699c00 x1875771511618944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879396.990610:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511618944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879396.990626:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879396.990633:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879396.990640:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511618944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879396.990671:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879396.990708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879396.990747:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879396/real 1788879396] req@ffffa0618869b800 x1875771511619072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879396.990764:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511619072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879396.990775:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879396.990779:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879396.990783:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879396.990808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879396.990836:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879396.990849:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879396/real 1788879396] req@ffffa06188698380 x1875771511619200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879396.990878:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511619200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879396.990887:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879396.990890:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879396.990893:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879397.629549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879397.629603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879397.885560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879398.653580:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879398.653586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879398.909596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879399.677474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879399.677674:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879399.933574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879400.701564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879400.701739:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879400.957581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879401.725554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879401.725658:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879401.981587:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879402.109473:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879402.109486:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879402.109493:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879402.109505:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4388 00000100:00080000:1.0:1788879402.109521:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879402.109530:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879402.109583:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511619328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879402.109634:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879402.109641:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879402.109645:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879402.109671:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4388 00000100:00080000:1.0:1788879402.109678:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879402.109684:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879402.109699:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879402.109722:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511619456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879402.109736:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879402.109740:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879402.109742:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879402.109751:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4388 00000100:00080000:1.0:1788879402.109756:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788879402.109761:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511619328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879402.109763:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879402.109787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879402.109830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511619584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879402.109847:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879402.109852:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879402.109854:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879402.109860:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4388 00000100:00080000:1.0:1788879402.109868:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879402.109873:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879402.109882:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788879402.109882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879402.109921:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511619712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879402.109935:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879402.109940:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879402.109944:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788879402.109952:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879402/real 1788879402] req@ffffa06188699f80 x1875771511619328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879402.109954:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4388 00000100:00080000:1.0:1788879402.109972:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879402.109984:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879402.109986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511619328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879402.109999:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1788879402.110006:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879402.110026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879402.110032:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511619328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879402.110044:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511619840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879402.110070:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879402.110112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879402.110126:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879402/real 1788879402] req@ffffa0618869a680 x1875771511619456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879402.110139:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511619456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879402.110148:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879402.110152:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879402.110156:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879402.110162:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879402.110187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879402.110196:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879402/real 1788879402] req@ffffa0618869bb80 x1875771511619584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879402.110208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511619584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879402.110216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879402.110219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879402.110222:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879402.110229:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879402.110246:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879402.110255:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879402/real 1788879402] req@ffffa0618869ad80 x1875771511619712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879402.110267:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511619712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879402.110275:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879402.110289:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879402.110294:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879402.110303:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879402.110333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879402.110350:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879402/real 1788879402] req@ffffa06188698000 x1875771511619840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879402.110366:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511619840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879402.110446:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879402.110452:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879402.110457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511619840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879402.749572:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879402.749695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879403.005509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879403.773544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879403.773612:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879404.029563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879404.797525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879404.797589:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879405.053569:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788879405.821555:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879405.821567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788879406.077540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879406.845490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879406.845590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879407.101497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879407.229463:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879407.229473:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879407.229479:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879407.229490:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4393 00000100:00080000:1.0:1788879407.229503:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879407.229511:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879407.229557:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511619968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879407.229584:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879407.229587:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879407.229589:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879407.229595:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4393 00000100:00080000:1.0:1788879407.229599:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879407.229604:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879407.229613:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879407.229623:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511620096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879407.229632:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879407.229636:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879407.229637:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788879407.229637:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511619968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879407.229645:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4393 00000100:00080000:1.0:1788879407.229648:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879407.229651:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879407.229654:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879407.229662:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511620224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879407.229668:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879407.229670:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879407.229672:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879407.229674:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4393 00000100:00080000:1.0:1788879407.229676:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879407.229678:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879407.229682:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879407.229689:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511620352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879407.229696:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879407.229698:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879407.229700:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879407.229702:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4393 00000100:00080000:1.0:1788879407.229704:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879407.229706:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879407.229709:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788879407.229710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879407.229719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511620480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879407.229755:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879407/real 1788879407] req@ffffa06188698000 x1875771511619968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879407.229777:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511619968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879407.229790:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879407.229793:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879407.229796:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511619968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879407.229819:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879407.229838:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879407.229847:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879407/real 1788879407] req@ffffa0618869ad80 x1875771511620096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879407.229855:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511620096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879407.229862:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879407.229864:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879407.229866:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879407.229871:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879407.229882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879407.229889:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879407/real 1788879407] req@ffffa0618869bb80 x1875771511620224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879407.229897:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511620224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879407.229902:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879407.229904:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879407.229906:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879407.229909:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879407.229920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879407.229926:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879407/real 1788879407] req@ffffa0618869a680 x1875771511620352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879407.229933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511620352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879407.229939:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879407.229940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879407.229942:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879407.229945:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879407.229955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879407.229960:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879407/real 1788879407] req@ffffa06188699f80 x1875771511620480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879407.229968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511620480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879407.229973:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879407.229975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879407.229976:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879407.869509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879407.869654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879408.125497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788879408.893555:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879408.893557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788879409.149594:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879409.917551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879409.917621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879410.173517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879410.941541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879410.941648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879411.197470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879411.965647:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879411.965650:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879412.221484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879412.349475:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879412.349485:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879412.349490:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879412.349500:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4399 00000100:00080000:1.0:1788879412.349512:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879412.349520:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879412.349558:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511620608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879412.349584:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879412.349587:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879412.349589:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879412.349595:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4399 00000100:00080000:1.0:1788879412.349598:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879412.349601:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879412.349611:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879412.349631:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511620736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879412.349639:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879412.349642:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879412.349643:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879412.349645:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4399 00000100:00080000:1.0:1788879412.349647:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879412.349649:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879412.349652:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879412.349658:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511620864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879412.349664:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879412.349667:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879412.349668:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879412.349672:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4399 00000100:00080000:1.0:1788879412.349675:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879412.349679:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879412.349682:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879412.349689:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511620992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879412.349697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879412.349700:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879412.349703:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879412.349707:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4399 00000100:00080000:1.0:1788879412.349709:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879412.349711:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879412.349714:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879412.349725:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511621120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879412.349749:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511620608:192.168.203.111@tcp:250:kworker.0 00000800:00000100:2.0:1788879412.349870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879412.349940:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879412/real 1788879412] req@ffffa06188698380 x1875771511620608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879412.349968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511620608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879412.349983:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879412.349990:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879412.349995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511620608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879412.350007:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879412.350039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879412.350056:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879412/real 1788879412] req@ffffa0618869b800 x1875771511620736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879412.350071:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511620736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879412.350082:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879412.350086:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879412.350090:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879412.350098:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879412.350118:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879412.350129:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879412/real 1788879412] req@ffffa06188699c00 x1875771511620864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879412.350143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511620864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879412.350153:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879412.350156:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879412.350160:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879412.350167:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879412.350187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879412.350198:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879412/real 1788879412] req@ffffa0618869aa00 x1875771511620992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879412.350230:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511620992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879412.350238:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879412.350241:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879412.350244:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511620992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879412.350281:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879412.350325:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879412.350341:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879412/real 1788879412] req@ffffa0618869b100 x1875771511621120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879412.350354:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511621120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879412.350363:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879412.350366:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879412.350370:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621120:192.168.203.111@tcp:8:kworker.0 00010000:00010000:0.0:1788879412.989587:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879412.989595:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788879413.245508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879414.013573:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879414.013655:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879414.269587:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879415.037592:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879415.037644:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879415.293556:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879416.061536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879416.061620:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879416.317555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879417.085545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879417.085700:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879417.341539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879417.469537:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879417.469552:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879417.469559:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879417.469577:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4404 00000100:00080000:1.0:1788879417.469596:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879417.469609:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879417.469671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511621248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879417.469739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879417.469745:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879417.469747:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879417.469754:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4404 00000100:00080000:1.0:1788879417.469759:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879417.469766:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879417.469781:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879417.469801:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511621376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879417.469819:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511621248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879417.469822:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879417.469835:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879417.469838:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879417.469844:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4404 00000100:00080000:1.0:1788879417.469848:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879417.469853:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879417.469859:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879417.469875:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511621504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879417.469905:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879417.469910:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879417.469913:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879417.469917:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4404 00000100:00080000:1.0:1788879417.469922:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879417.469926:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879417.469934:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788879417.469950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879417.469951:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511621632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879417.469993:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879417.469999:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879417.470002:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879417.470008:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4404 00000100:00080000:1.0:1788879417.470017:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879417.470022:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879417.470029:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879417.470053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879417/real 1788879417] req@ffffa0618869b100 x1875771511621248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879417.470054:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511621760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879417.470133:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511621248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879417.470151:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879417.470156:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879417.470162:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511621248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879417.470219:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879417.470260:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879417.470340:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879417/real 1788879417] req@ffffa0618869aa00 x1875771511621376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879417.470357:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511621376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879417.470380:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879417.470384:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879417.470388:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879417.470397:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879417.470508:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879417.470525:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879417/real 1788879417] req@ffffa06188699c00 x1875771511621504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879417.470541:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511621504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879417.470551:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879417.470555:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879417.470558:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879417.470565:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879417.470585:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879417.470596:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879417/real 1788879417] req@ffffa0618869b800 x1875771511621632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879417.470609:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511621632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879417.470618:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879417.470621:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879417.470624:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879417.470630:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879417.470646:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879417.470656:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879417/real 1788879417] req@ffffa06188698380 x1875771511621760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879417.470678:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511621760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879417.470723:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879417.470730:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879417.470737:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511621760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879418.109536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879418.109592:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879418.365479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879419.133534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879419.133693:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879419.389556:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879420.157498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879420.157560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879420.413546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879421.181442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879421.181515:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879421.437468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879422.205515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879422.205525:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879422.461484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879422.589483:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879422.589492:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879422.589496:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879422.589506:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4409 00000100:00080000:2.0:1788879422.589517:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879422.589525:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879422.589570:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511621888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879422.589593:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879422.589596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879422.589597:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879422.589612:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4409 00000100:00080000:2.0:1788879422.589615:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879422.589619:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879422.589629:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879422.589640:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511621888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879422.589640:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511622016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879422.589654:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879422.589656:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879422.589658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879422.589661:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4409 00000100:00080000:2.0:1788879422.589663:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879422.589665:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879422.589669:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879422.589678:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511622144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879422.589685:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879422.589688:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879422.589689:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879422.589692:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4409 00000100:00080000:2.0:1788879422.589694:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879422.589696:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879422.589699:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879422.589707:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511622272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879422.589712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879422.589722:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879422.589726:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879422.589728:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879422.589731:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4409 00000100:00080000:2.0:1788879422.589733:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879422.589736:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879422.589740:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879422.589747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511622400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879422.589752:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879422/real 1788879422] req@ffffa061888af480 x1875771511621888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879422.589772:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511621888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879422.589782:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879422.589785:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879422.589788:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511621888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879422.589810:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879422.589824:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879422.589832:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879422/real 1788879422] req@ffffa061888aca80 x1875771511622016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879422.589840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511622016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879422.589856:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879422.589858:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879422.589860:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879422.589864:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879422.589874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879422.589880:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879422/real 1788879422] req@ffffa061888adf80 x1875771511622144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879422.589888:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511622144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879422.589893:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879422.589894:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879422.589896:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879422.589899:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879422.589909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879422.589915:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879422/real 1788879422] req@ffffa061888ac000 x1875771511622272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879422.589922:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511622272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879422.589928:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879422.589929:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879422.589931:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879422.589934:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879422.589943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879422.589949:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879422/real 1788879422] req@ffffa061888ad500 x1875771511622400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879422.589956:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511622400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879422.589965:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879422.589967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879422.589969:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879423.229442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879423.229508:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879423.485486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879424.253528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879424.253635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879424.509512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879425.277475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879425.277538:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879425.533491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879426.301564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879426.301717:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879426.557519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879427.325482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879427.325593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879427.581457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879427.709544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879427.709577:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879427.709583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879427.709597:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4414 00000100:00080000:2.0:1788879427.709613:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879427.709648:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879427.709726:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511622528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879427.709777:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879427.709784:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879427.709788:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879427.709797:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4414 00000100:00080000:2.0:1788879427.709805:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879427.709813:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879427.709830:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879427.709848:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511622528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879427.709855:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511622656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879427.709885:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879427.709890:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879427.709894:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879427.709899:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4414 00000100:00080000:2.0:1788879427.709922:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879427.709927:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879427.709934:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879427.709949:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511622784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879427.709963:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879427.709968:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788879427.709971:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879427.709971:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879427.709977:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4414 00000100:00080000:2.0:1788879427.709982:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879427.709988:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879427.709995:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879427.710011:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511622912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879427.710021:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879427/real 1788879427] req@ffffa061888af100 x1875771511622528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879427.710026:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879427.710032:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879427.710035:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879427.710040:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4414 00000100:00080000:2.0:1788879427.710045:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879427.710050:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511622528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879427.710051:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879427.710059:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879427.710069:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788879427.710073:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511623040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879427.710075:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879427.710083:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511622528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879427.710119:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879427.710157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879427.710174:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879427/real 1788879427] req@ffffa061888ae300 x1875771511622656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879427.710188:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511622656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879427.710231:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879427.710235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879427.710266:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879427.710275:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879427.710305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879427.710320:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879427/real 1788879427] req@ffffa061888aea00 x1875771511622784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879427.710335:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511622784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879427.710345:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879427.710349:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879427.710353:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879427.710360:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879427.710379:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879427.710390:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879427/real 1788879427] req@ffffa061888afb80 x1875771511622912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879427.710439:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511622912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879427.710456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879427.710460:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879427.710465:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511622912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879427.710472:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879427.710495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879427.710509:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879427/real 1788879427] req@ffffa061888adc00 x1875771511623040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879427.710533:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511623040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879427.710542:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879427.710546:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879427.710566:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879428.349567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879428.349657:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879428.605513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879429.373555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879429.373605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879429.629510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879430.397520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879430.397596:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879430.653493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879431.421521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879431.421688:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879431.677485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879432.445511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879432.445643:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879432.701528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879432.829547:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879432.829559:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879432.829563:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879432.829577:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4419 00000100:00080000:1.0:1788879432.829594:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879432.829629:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879432.829676:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511623168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879432.829716:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879432.829720:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879432.829721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879432.829725:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4419 00000100:00080000:1.0:1788879432.829730:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879432.829734:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879432.829746:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879432.829760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511623296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879432.829771:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879432.829774:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879432.829775:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879432.829782:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4419 00000100:00080000:1.0:1788879432.829790:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879432.829794:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511623168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879432.829795:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879432.829810:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879432.829824:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511623424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879432.829834:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879432.829839:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879432.829841:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879432.829845:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4419 00000100:00080000:1.0:1788879432.829866:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879432.829872:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879432.829879:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879432.829894:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511623552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879432.829911:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879432.829924:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788879432.829927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879432.829929:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879432.829942:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4419 00000100:00080000:1.0:1788879432.829950:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879432.829958:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879432.829968:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879432.829991:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879432/real 1788879432] req@ffffa06188699f80 x1875771511623168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879432.829995:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511623680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879432.830061:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511623168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879432.830087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879432.830094:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879432.830100:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511623168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879432.830151:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879432.830237:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879432.830273:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879432/real 1788879432] req@ffffa0618869a680 x1875771511623296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879432.830301:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511623296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879432.830314:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879432.830319:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879432.830323:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879432.830343:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879432.830377:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879432.830441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879432/real 1788879432] req@ffffa0618869bb80 x1875771511623424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879432.830457:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511623424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879432.830468:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879432.830472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879432.830476:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879432.830485:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879432.830512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879432.830525:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879432/real 1788879432] req@ffffa0618869ad80 x1875771511623552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879432.830541:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511623552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879432.830551:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879432.830555:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879432.830558:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879432.830566:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879432.830586:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879432.830606:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879432/real 1788879432] req@ffffa06188698000 x1875771511623680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879432.830620:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511623680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879432.830630:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879432.830634:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879432.830637:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879433.469500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879433.469582:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879433.725462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879434.493478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879434.493614:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879434.749443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879435.517488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879435.517587:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879435.773442:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879436.541444:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879436.541564:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879436.797477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879437.565490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879437.565649:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879437.821475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879437.949483:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879437.949492:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879437.949497:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879437.949508:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4424 00000100:00080000:1.0:1788879437.949519:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879437.949526:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879437.949563:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511623808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879437.949583:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879437.949586:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879437.949588:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879437.949591:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4424 00000100:00080000:1.0:1788879437.949595:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879437.949598:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879437.949608:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879437.949620:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511623808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879437.949620:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511623936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879437.949635:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879437.949637:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879437.949638:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879437.949640:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4424 00000100:00080000:1.0:1788879437.949642:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879437.949645:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879437.949648:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879437.949655:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511624064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879437.949661:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879437.949663:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879437.949664:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879437.949667:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4424 00000100:00080000:1.0:1788879437.949679:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879437.949682:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879437.949686:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879437.949694:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511624192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879437.949712:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879437.949715:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879437.949716:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879437.949720:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4424 00000100:00080000:1.0:1788879437.949723:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879437.949725:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623936:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879437.949728:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879437.949741:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879437.949752:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511624320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879437.949770:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879437.949789:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879437.949804:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879438.589515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879438.589652:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879438.845513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879439.613566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879439.613663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879439.869594:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879440.637509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879440.637680:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879440.893500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879441.021660:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879441.021700:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879441.021715:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879441.021749:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879441.021757:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879441.021767:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879441.021778:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879441.021799:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879437/real 1788879441] req@ffffa0618869ad80 x1875771511623936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879441.021830:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511623936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879441.021844:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879441.021849:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879441.021854:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511623936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879441.021863:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879437/real 1788879441] req@ffffa0618869bb80 x1875771511624064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879441.021877:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511624064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879441.021886:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879441.021889:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879441.021893:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879441.021900:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879437/real 1788879441] req@ffffa0618869a680 x1875771511624192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879441.021913:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511624192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879441.021922:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879441.021926:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879441.021929:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879441.021937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879437/real 1788879441] req@ffffa06188698000 x1875771511623808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879441.021950:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511623808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879441.021958:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879441.021962:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879441.021965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511623808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879441.021992:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879437/real 1788879441] req@ffffa06188699f80 x1875771511624320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879441.022005:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511624320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879441.022014:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879441.022017:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879441.022021:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879441.661501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879441.661623:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879441.917563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879442.685449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879442.685533:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879442.941550:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879443.069591:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879443.069603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879443.069608:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879443.069621:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4429 00000100:00080000:2.0:1788879443.069636:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879443.069647:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879443.069709:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511624448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879443.069745:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879443.069749:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879443.069751:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879443.069756:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4429 00000100:00080000:2.0:1788879443.069762:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879443.069778:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879443.069791:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879443.069797:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511624448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879443.069819:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511624576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879443.069838:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879443.069845:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879443.069848:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879443.069854:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4429 00000100:00080000:2.0:1788879443.069861:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879443.069868:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879443.069875:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879443.069885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879443.069894:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511624704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879443.069912:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879443.069919:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879443.069922:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879443.069928:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4429 00000100:00080000:2.0:1788879443.069935:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879443.069941:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879443.069948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879443/real 1788879443] req@ffffa061888aed80 x1875771511624448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879443.069952:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879443.069979:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511624832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879443.069990:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511624448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879443.070003:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879443.070011:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879443.070013:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879443.070022:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879443.070023:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879443.070034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511624448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879443.070038:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4429 00000100:00080000:2.0:1788879443.070047:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879443.070056:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879443.070064:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879443.070072:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879443.070117:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511624960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879443.070139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879443.070165:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879443/real 1788879443] req@ffffa061888ac700 x1875771511624576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879443.070185:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511624576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879443.070199:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879443.070204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879443.070208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879443.070217:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879443.070248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879443.070265:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879443/real 1788879443] req@ffffa061888ac380 x1875771511624704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879443.070282:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511624704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879443.070302:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879443.070306:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879443.070310:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879443.070319:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879443.070346:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879443.070361:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879443/real 1788879443] req@ffffa061888ad880 x1875771511624832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879443.070378:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511624832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879443.070430:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879443.070436:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879443.070441:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879443.070466:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879443.070497:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879443.070515:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879443/real 1788879443] req@ffffa061888ae680 x1875771511624960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879443.070533:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511624960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879443.070547:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879443.070551:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879443.070557:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511624960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879443.709493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879443.709550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879443.965513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879444.733499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879444.733643:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879444.989507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879445.757536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879445.757687:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879446.013477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879446.781537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879446.781784:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879447.037502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879447.805486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879447.805530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879448.061452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788879448.189625:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879448.189634:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788879448.189637:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879448.189647:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4434 00000100:00080000:0.0:1788879448.189659:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879448.189669:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879448.189713:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511625088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879448.189743:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879448.189747:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788879448.189749:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879448.189753:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4434 00000100:00080000:0.0:1788879448.189757:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879448.189761:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879448.189771:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879448.189786:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511625216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879448.189787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511625088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788879448.189802:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879448.189805:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788879448.189807:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879448.189810:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4434 00000100:00080000:0.0:1788879448.189824:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879448.189826:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879448.189848:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879448.189868:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511625344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879448.189881:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879448.189886:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788879448.189889:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879448.189894:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4434 00000100:00080000:0.0:1788879448.189899:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879448.189904:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788879448.189913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:0.0:1788879448.189914:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879448.189957:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511625472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879448.189969:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879448.189974:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879448.189977:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879448.189982:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4434 00000100:00080000:0.0:1788879448.189987:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879448.189992:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879448.189998:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879448.190017:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511625600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879448.190017:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879448/real 1788879448] req@ffffa06198aff800 x1875771511625088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879448.190097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511625088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879448.190107:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879448.190111:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879448.190115:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511625088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879448.190136:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879448.190154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879448.190172:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879448/real 1788879448] req@ffffa06198affb80 x1875771511625216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879448.190180:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511625216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879448.190186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879448.190207:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879448.190211:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879448.190218:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879448.190238:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879448.190250:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879448/real 1788879448] req@ffffa06198afd880 x1875771511625344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879448.190263:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511625344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879448.190272:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879448.190275:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879448.190279:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879448.190285:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879448.190303:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879448.190325:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879448/real 1788879448] req@ffffa06198aff480 x1875771511625472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879448.190338:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511625472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879448.190347:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879448.190350:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879448.190353:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879448.190359:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879448.190376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879448.190394:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879448/real 1788879448] req@ffffa06198afed80 x1875771511625600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879448.190439:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511625600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879448.190445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879448.190447:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879448.190449:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879448.829490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879448.829567:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879449.085491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879449.853469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879449.853526:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879450.109459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879450.877492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879450.877600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879451.133522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879451.901532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879451.901614:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879452.157504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879452.925502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879452.925635:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879453.181490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788879453.309545:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879453.309556:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788879453.309562:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879453.309574:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4440 00000100:00080000:0.0:1788879453.309587:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879453.309597:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879453.309642:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511625728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879453.309684:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879453.309690:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788879453.309694:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879453.309701:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4440 00000100:00080000:0.0:1788879453.309707:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879453.309713:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879453.309728:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879453.309751:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511625856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879453.309765:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879453.309771:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1788879453.309771:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511625728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788879453.309778:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879453.309786:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4440 00000100:00080000:0.0:1788879453.309790:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879453.309795:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879453.309801:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879453.309814:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511625984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879453.309825:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879453.309829:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788879453.309831:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879453.309835:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4440 00000100:00080000:0.0:1788879453.309838:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879453.309842:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879453.309847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879453.309859:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511626112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788879453.309867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788879453.309878:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879453.309883:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879453.309886:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879453.309894:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4440 00000100:00080000:0.0:1788879453.309902:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879453.309909:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879453.309918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879453.309919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879453/real 1788879453] req@ffffa06198afed80 x1875771511625728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879453.309948:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511626240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879453.309951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511625728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879453.309996:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879453.310003:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879453.310009:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511625728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879453.310061:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879453.310090:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879453.310105:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879453/real 1788879453] req@ffffa06198aff480 x1875771511625856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879453.310118:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511625856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879453.310127:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879453.310131:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879453.310134:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879453.310140:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879453.310157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879453.310166:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879453/real 1788879453] req@ffffa06198afd880 x1875771511625984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879453.310178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511625984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879453.310185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879453.310188:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879453.310191:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511625984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879453.310196:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879453.310212:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879453.310221:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879453/real 1788879453] req@ffffa06198affb80 x1875771511626112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879453.310231:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511626112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879453.310240:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879453.310243:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879453.310246:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879453.310251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879453.310266:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879453.310274:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879453/real 1788879453] req@ffffa06198aff800 x1875771511626240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879453.310285:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511626240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879453.310293:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879453.310296:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879453.310299:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879453.949479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879453.949572:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879454.205486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879454.973530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879454.973647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879455.229487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879455.997506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879455.997682:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879456.253498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879457.021543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879457.021619:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879457.277530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879458.045488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879458.045602:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879458.301503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788879458.429554:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879458.429565:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788879458.429570:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879458.429582:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4445 00000100:00080000:0.0:1788879458.429594:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879458.429603:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879458.429647:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511626368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879458.429695:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879458.429698:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788879458.429700:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879458.429704:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4445 00000100:00080000:0.0:1788879458.429708:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879458.429711:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879458.429721:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879458.429732:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511626496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879458.429740:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879458.429743:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788879458.429745:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879458.429748:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4445 00000100:00080000:0.0:1788879458.429751:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879458.429754:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879458.429757:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879458.429766:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511626624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879458.429774:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879458.429777:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788879458.429779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879458.429782:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4445 00000100:00080000:0.0:1788879458.429785:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879458.429788:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879458.429792:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879458.429802:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511626368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879458.429805:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511626752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879458.429824:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879458.429826:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879458.429828:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879458.429831:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4445 00000100:00080000:0.0:1788879458.429835:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879458.429837:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788879458.429841:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879458.429876:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511626880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788879458.429917:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879458.429963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879458/real 1788879458] req@ffffa06198afdf80 x1875771511626368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879458.430010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511626368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879458.430044:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879458.430050:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879458.430055:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511626368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879458.430066:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879458.430103:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879458.430124:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879458/real 1788879458] req@ffffa06198afdc00 x1875771511626496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879458.430140:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511626496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879458.430149:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879458.430152:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879458.430156:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879458.430162:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788879458.430183:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879458.430193:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879458/real 1788879458] req@ffffa06198afc700 x1875771511626624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879458.430236:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511626624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879458.430243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879458.430246:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879458.430248:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879458.430309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879458.430329:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879458.430339:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879458/real 1788879458] req@ffffa06198afc000 x1875771511626752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879458.430351:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511626752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879458.430358:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879458.430361:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879458.430364:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879458.430369:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788879458.430382:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879458.430390:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879458/real 1788879458] req@ffffa06198afea00 x1875771511626880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879458.430434:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511626880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879458.430449:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879458.430452:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879458.430455:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511626880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879459.069482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879459.069655:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879459.325504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879460.093491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879460.093577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879460.349517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879461.117468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879461.117550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879461.373475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879462.141499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879462.141580:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879462.397474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879463.165436:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879463.165533:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879463.421485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879463.549489:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879463.549499:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879463.549504:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879463.549516:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4450 00000100:00080000:2.0:1788879463.549529:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879463.549539:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879463.549579:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511627008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879463.549604:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879463.549608:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879463.549610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879463.549615:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4450 00000100:00080000:2.0:1788879463.549620:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879463.549625:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879463.549636:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879463.549641:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511627008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879463.549649:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511627136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879463.549660:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879463.549663:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879463.549665:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879463.549668:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4450 00000100:00080000:2.0:1788879463.549672:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879463.549675:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879463.549680:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879463.549692:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511627264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879463.549701:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879463.549705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879463.549706:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879463.549710:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4450 00000100:00080000:2.0:1788879463.549713:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879463.549716:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879463.549720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788879463.549722:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879463.549732:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511627392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879463.549745:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879463.549749:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879463.549750:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879463.549753:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4450 00000100:00100000:0.0:1788879463.549756:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879463/real 1788879463] req@ffffa061888ad180 x1875771511627008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879463.549756:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879463.549760:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879463.549765:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879463.549775:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511627008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879463.549775:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511627520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879463.549788:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879463.549792:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879463.549797:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511627008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879463.549822:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879463.549843:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879463.549852:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879463/real 1788879463] req@ffffa061888ace00 x1875771511627136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879463.549860:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511627136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879463.549866:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879463.549868:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879463.549871:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879463.549875:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879463.549887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879463.549893:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879463/real 1788879463] req@ffffa061888ae680 x1875771511627264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879463.549900:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511627264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879463.549905:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879463.549907:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879463.549909:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879463.549912:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879463.549922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879463.549928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879463/real 1788879463] req@ffffa061888ad880 x1875771511627392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879463.549935:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511627392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879463.549940:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879463.549959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879463.549962:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879463.549966:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879463.549980:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879463.549989:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879463/real 1788879463] req@ffffa061888ac380 x1875771511627520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879463.549996:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511627520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879463.550002:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879463.550003:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879463.550005:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627520:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788879464.189558:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879464.189558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788879464.445590:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879465.213504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879465.213535:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879465.469471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879466.237543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879466.237626:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879466.493535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879467.261543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879467.261701:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879467.517519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879468.285540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879468.285681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879468.541583:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879468.669585:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879468.669607:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879468.669617:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879468.669636:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4455 00000100:00080000:1.0:1788879468.669655:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879468.669667:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879468.669733:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511627648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879468.669775:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879468.669781:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879468.669784:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879468.669789:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4455 00000100:00080000:1.0:1788879468.669794:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879468.669799:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879468.669813:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879468.669831:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511627776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879468.669848:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879468.669851:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511627648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879468.669856:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879468.669864:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879468.669873:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4455 00000100:00080000:1.0:1788879468.669877:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879468.669881:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879468.669887:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879468.669904:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511627904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879468.669949:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879468.669955:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879468.669958:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879468.669964:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4455 00000100:00080000:1.0:1788879468.669974:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879468.669980:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879468.669989:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879468.669994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879468.670014:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511628032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879468.670029:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879468.670034:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879468.670037:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879468.670042:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4455 00000100:00080000:1.0:1788879468.670047:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879468.670052:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879468.670059:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879468.670064:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879468/real 1788879468] req@ffffa06188698380 x1875771511627648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879468.670078:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511628160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879468.670120:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511627648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879468.670139:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879468.670145:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879468.670150:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511627648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879468.670195:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879468.670245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879468.670271:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879468/real 1788879468] req@ffffa0618869b800 x1875771511627776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879468.670299:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511627776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879468.670312:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879468.670316:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879468.670321:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879468.670329:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879468.670358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879468.670372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879468/real 1788879468] req@ffffa06188699c00 x1875771511627904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879468.670421:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511627904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879468.670433:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879468.670437:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879468.670441:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511627904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879468.670448:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879468.670474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879468.670487:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879468/real 1788879468] req@ffffa0618869aa00 x1875771511628032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879468.670502:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511628032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879468.670520:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879468.670524:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879468.670528:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879468.670535:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879468.670554:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879468.670565:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879468/real 1788879468] req@ffffa0618869b100 x1875771511628160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879468.670579:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511628160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879468.670589:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879468.670593:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879468.670596:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879469.309585:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879469.309604:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879469.565605:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879470.333554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879470.333722:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879470.589495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879471.357587:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879471.357683:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879471.613620:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879472.381611:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879472.381666:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879472.637636:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879473.405590:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879473.405629:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879473.661507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879473.789462:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879473.789472:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879473.789476:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879473.789485:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4460 00000100:00080000:1.0:1788879473.789497:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879473.789505:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879473.789554:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511628288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879473.789574:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879473.789577:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879473.789579:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879473.789583:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4460 00000100:00080000:1.0:1788879473.789586:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879473.789590:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879473.789599:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879473.789613:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511628416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879473.789624:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511628288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879473.789625:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879473.789633:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879473.789637:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879473.789644:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4460 00000100:00080000:1.0:1788879473.789648:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879473.789654:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879473.789661:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879473.789676:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511628544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879473.789687:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879473.789691:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879473.789693:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879473.789696:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4460 00000100:00080000:1.0:1788879473.789703:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879473.789708:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879473.789714:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879473.789717:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879473.789731:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511628672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879473.789743:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879473.789747:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879473.789750:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879473.789755:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4460 00000100:00080000:1.0:1788879473.789759:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879473.789764:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879473.789772:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879473/real 1788879473] req@ffffa0618869b100 x1875771511628288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879473.789773:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879473.789791:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511628800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879473.789804:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511628288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879473.789822:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879473.789827:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879473.789839:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511628288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879473.789865:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879473.789899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879473.789910:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879473/real 1788879473] req@ffffa0618869aa00 x1875771511628416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879473.789919:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511628416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879473.789926:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879473.789928:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879473.789930:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879473.789936:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879473.789949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879473.789955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879473/real 1788879473] req@ffffa06188699c00 x1875771511628544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879473.789963:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511628544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879473.789969:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879473.789971:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879473.789973:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879473.789978:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879473.789990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879473.789996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879473/real 1788879473] req@ffffa0618869b800 x1875771511628672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879473.790004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511628672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879473.790010:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879473.790012:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879473.790014:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879473.790017:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879473.790028:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879473.790035:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879473/real 1788879473] req@ffffa06188698380 x1875771511628800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879473.790042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511628800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879473.790047:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879473.790049:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879473.790051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511628800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879474.429543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879474.429653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879474.685490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879475.453529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879475.453638:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879475.709484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879476.477529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879476.477639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879476.733500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879477.501494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879477.501633:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879477.757495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879478.525506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879478.525650:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879478.781522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879478.909595:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879478.909608:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879478.909613:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879478.909627:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4465 00000100:00080000:2.0:1788879478.909641:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879478.909651:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879478.909698:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511628928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879478.909730:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879478.909735:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879478.909737:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879478.909742:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4465 00000100:00080000:2.0:1788879478.909746:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879478.909750:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879478.909763:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879478.909780:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511629056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879478.909796:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511628928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879478.909797:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879478.909806:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879478.909811:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879478.909818:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4465 00000100:00080000:2.0:1788879478.909822:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879478.909826:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879478.909831:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879478.909880:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511629184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879478.909896:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879478.909901:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879478.909903:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879478.909907:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4465 00000100:00080000:2.0:1788879478.909913:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879478.909918:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879478.909928:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879478.909930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879478.909970:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511629312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879478.909988:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879478.909993:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879478/real 1788879478] req@ffffa061888ac700 x1875771511628928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879478.909997:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879478.910005:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879478.910019:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4465 00000100:00080000:2.0:1788879478.910027:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879478.910029:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511628928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879478.910037:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879478.910053:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879478.910058:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879478.910065:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879478.910074:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511629440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879478.910075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511628928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879478.910112:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879478.910141:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879478.910156:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879478/real 1788879478] req@ffffa061888aed80 x1875771511629056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879478.910171:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511629056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879478.910180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879478.910184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879478.910188:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879478.910194:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879478.910211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879478.910220:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879478/real 1788879478] req@ffffa061888adc00 x1875771511629184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879478.910240:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511629184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879478.910249:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879478.910252:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879478.910255:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879478.910261:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879478.910278:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879478.910295:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879478/real 1788879478] req@ffffa061888afb80 x1875771511629312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879478.910308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511629312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879478.910316:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879478.910319:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879478.910322:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879478.910328:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879478.910343:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879478.910352:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879478/real 1788879478] req@ffffa061888aea00 x1875771511629440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879478.910364:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511629440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879478.910380:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879478.910411:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879478.910414:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879479.549522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879479.549676:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879479.805511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879480.573567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879480.573657:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879480.829504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879481.597596:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879481.597701:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879481.853546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879482.621503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879482.621698:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879482.877540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879483.645567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879483.645633:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879483.901570:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879484.029739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879484.029759:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879484.029767:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879484.029787:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4470 00000100:00080000:1.0:1788879484.029834:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879484.029858:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879484.029938:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511629568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879484.030001:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879484.030009:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879484.030013:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879484.030023:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4470 00000100:00080000:1.0:1788879484.030031:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879484.030039:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879484.030056:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879484.030106:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511629696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879484.030109:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511629568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879484.030138:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879484.030161:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879484.030164:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879484.030171:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4470 00000100:00080000:1.0:1788879484.030177:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879484.030182:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879484.030189:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879484.030207:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511629824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879484.030220:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879484.030226:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879484.030228:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879484.030232:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4470 00000100:00080000:1.0:1788879484.030238:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879484.030244:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879484.030256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879484.030256:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879484.030278:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511629952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879484.030293:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879484.030298:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879484.030301:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879484.030305:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4470 00000100:00080000:1.0:1788879484.030310:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879484.030316:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879484.030323:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879484.030339:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879484/real 1788879484] req@ffffa06188699f80 x1875771511629568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879484.030343:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511630080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879484.030467:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511629568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879484.030487:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879484.030494:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879484.030500:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511629568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879484.030540:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879484.030599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879484.030616:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879484/real 1788879484] req@ffffa06188698000 x1875771511629696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879484.030632:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511629696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879484.030681:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879484.030688:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879484.030695:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879484.030708:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879484.030748:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879484.030771:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879484/real 1788879484] req@ffffa0618869a680 x1875771511629824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879484.030797:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511629824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879484.030837:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879484.030842:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879484.030847:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879484.030856:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879484.030902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879484.030919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879484/real 1788879484] req@ffffa0618869bb80 x1875771511629952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879484.030935:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511629952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879484.030945:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879484.030949:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879484.030954:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511629952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879484.030963:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879484.030984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879484.030997:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879484/real 1788879484] req@ffffa0618869ad80 x1875771511630080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879484.031011:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511630080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879484.031021:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879484.031024:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879484.031028:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879484.669505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879484.669534:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879484.925483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879485.693444:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879485.693551:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879485.949475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879486.717507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879486.717642:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879486.973548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879487.741477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879487.741548:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879487.997484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879488.765481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879488.765586:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879489.021491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879489.149482:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879489.149494:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879489.149499:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879489.149513:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4475 00000100:00080000:1.0:1788879489.149525:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879489.149535:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879489.149580:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511630208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879489.149621:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879489.149625:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879489.149628:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879489.149634:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4475 00000100:00080000:1.0:1788879489.149640:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879489.149645:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879489.149652:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511630208:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879489.149660:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879489.149688:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511630336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879489.149701:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879489.149705:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879489.149708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879489.149712:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4475 00000100:00080000:1.0:1788879489.149717:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879489.149721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879489.149725:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879489.149733:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879489.149747:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511630464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879489.149755:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879489/real 1788879489] req@ffffa0618869ad80 x1875771511630208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879489.149780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879489.149785:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879489.149789:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879489.149795:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4475 00000100:00080000:1.0:1788879489.149801:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879489.149803:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511630208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879489.149809:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879489.149820:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879489.149823:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879489.149829:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879489.149832:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511630208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879489.149838:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511630592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879489.149850:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879489.149854:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879489.149856:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879489.149859:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630336:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879489.149864:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4475 00000100:00080000:1.0:1788879489.149869:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879489.149874:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879489.149880:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879489.149885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879489.149901:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511630720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879489.149902:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879489/real 1788879489] req@ffffa0618869bb80 x1875771511630336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879489.149944:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511630336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879489.149956:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879489.149961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879489.149965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879489.149970:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879489.149988:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879489.149996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879489/real 1788879489] req@ffffa0618869a680 x1875771511630464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879489.150004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511630464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879489.150009:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879489.150011:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879489.150013:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879489.150017:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879489.150027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879489.150033:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879489/real 1788879489] req@ffffa06188698000 x1875771511630592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879489.150040:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511630592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879489.150046:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879489.150048:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879489.150049:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879489.150061:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879489.150071:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879489.150077:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879489/real 1788879489] req@ffffa06188699f80 x1875771511630720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879489.150084:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511630720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879489.150089:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879489.150091:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879489.150093:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879489.789463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879489.789531:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879490.045474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879490.813453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879490.813521:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879491.069523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879491.837537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879491.837689:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879492.093497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879492.861516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879492.861699:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879493.117470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879493.885498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879493.885563:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879494.141468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879494.269463:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879494.269472:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879494.269476:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879494.269485:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4480 00000100:00080000:1.0:1788879494.269496:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879494.269503:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879494.269574:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511630848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879494.269604:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879494.269609:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879494.269611:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879494.269616:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4480 00000100:00080000:1.0:1788879494.269621:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879494.269626:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879494.269637:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879494.269650:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511630848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879494.269652:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511630976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879494.269668:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879494.269671:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879494.269673:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879494.269676:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4480 00000100:00080000:1.0:1788879494.269679:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879494.269683:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879494.269687:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879494.269697:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511631104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879494.269706:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879494.269710:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879494.269711:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879494.269715:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4480 00000100:00080000:1.0:1788879494.269751:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879494.269767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879494.269768:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879494.269791:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879494.269813:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511631232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879494.269825:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879494/real 1788879494] req@ffffa06188698380 x1875771511630848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879494.269831:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879494.269840:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879494.269844:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879494.269863:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4480 00000100:00080000:1.0:1788879494.269871:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879494.269878:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879494.269884:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879494.269889:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511630848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879494.269902:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788879494.269903:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511631360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879494.269911:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879494.269922:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511630848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879494.269951:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879494.269975:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879494.269986:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879494/real 1788879494] req@ffffa0618869b800 x1875771511630976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879494.269998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511630976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879494.270016:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879494.270019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879494.270022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511630976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879494.270028:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879494.270044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879494.270053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879494/real 1788879494] req@ffffa06188699c00 x1875771511631104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879494.270063:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511631104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879494.270086:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879494.270089:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879494.270092:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879494.270096:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879494.270110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879494.270118:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879494/real 1788879494] req@ffffa0618869aa00 x1875771511631232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879494.270128:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511631232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879494.270135:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879494.270137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879494.270140:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879494.270145:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879494.270158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879494.270165:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879494/real 1788879494] req@ffffa0618869b100 x1875771511631360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879494.270174:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511631360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879494.270181:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879494.270184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879494.270187:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879494.909471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879494.909613:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879495.165501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879495.933491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879495.933627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879496.189512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879496.957591:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879496.957666:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879497.213481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879497.981556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879497.981571:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879498.237499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879499.005597:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879499.005688:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879499.261534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879499.389598:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879499.389629:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879499.389637:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879499.389652:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4486 00000100:00080000:2.0:1788879499.389669:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879499.389682:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879499.389760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511631488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879499.389803:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879499.389810:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879499.389813:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879499.389820:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4486 00000100:00080000:2.0:1788879499.389828:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879499.389837:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879499.389865:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879499.389871:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511631488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879499.389893:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511631616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879499.389915:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879499.389921:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879499.389924:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879499.389930:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4486 00000100:00080000:2.0:1788879499.389937:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879499.389945:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879499.389954:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879499.389962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879499.389982:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511631744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879499.390002:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879499.390008:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788879499.390013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879499/real 1788879499] req@ffffa061888ae300 x1875771511631488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879499.390014:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879499.390031:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4486 00000100:00080000:2.0:1788879499.390044:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879499.390055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511631488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879499.390056:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879499.390083:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879499.390087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879499.390096:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879499.390101:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511631488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879499.390110:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511631872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879499.390130:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879499.390132:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631616:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788879499.390146:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879499.390152:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879499.390160:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4486 00000100:00080000:2.0:1788879499.390168:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879499.390172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879499.390180:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879499.390190:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879499.390192:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879499/real 1788879499] req@ffffa061888af100 x1875771511631616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879499.390210:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511631616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879499.390219:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511632000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879499.390225:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879499.390238:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879499.390250:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879499.390264:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879499.390286:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879499.390296:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879499/real 1788879499] req@ffffa061888ad500 x1875771511631744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879499.390308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511631744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879499.390316:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879499.390319:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879499.390322:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879499.390335:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879499.390351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879499.390360:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879499/real 1788879499] req@ffffa061888ac000 x1875771511631872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879499.390371:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511631872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879499.390407:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879499.390410:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879499.390414:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511631872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879499.390420:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879499.390439:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879499.390451:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879499/real 1788879499] req@ffffa061888adf80 x1875771511632000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879499.390463:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511632000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879499.390472:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879499.390475:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879499.390477:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879500.029466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879500.029588:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879500.285517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879501.053484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879501.053521:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879501.309493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879502.077507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879502.077574:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879502.333480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879503.101452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879503.101624:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879503.357462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879504.125519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879504.125652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879504.381483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879504.509586:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879504.509612:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879504.509620:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879504.509635:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4491 00000100:00080000:3.0:1788879504.509652:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879504.509692:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879504.509750:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511632128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879504.509810:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879504.509816:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879504.509820:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879504.509826:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4491 00000100:00080000:3.0:1788879504.509834:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879504.509839:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879504.509855:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879504.509873:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511632256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879504.509890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879504.509896:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879504.509900:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879504.509907:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4491 00000100:00080000:3.0:1788879504.509914:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879504.509917:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511632128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879504.509922:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879504.509936:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879504.509953:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511632384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879504.509967:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879504.509972:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879504.509975:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879504.509980:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4491 00000100:00080000:3.0:1788879504.509984:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879504.509989:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879504.509996:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879504.510010:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511632512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879504.510023:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879504.510027:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879504.510030:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879504.510045:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4491 00000100:00080000:3.0:1788879504.510050:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879504.510055:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879504.510062:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879504.510078:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511632640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879504.510109:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879504.510168:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879504.510213:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879504.510266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879505.149570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879505.149753:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879505.405522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879506.173540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879506.173829:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879506.429530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879507.197480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879507.197692:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879507.453527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788879507.581959:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788879507.582026:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788879507.582052:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879507.582109:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879507.582121:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879507.582141:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879507.582164:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879507.582197:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879504/real 1788879507] req@ffffa06188870380 x1875771511632384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879507.582251:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511632384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879507.582279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879507.582291:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879507.582300:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879507.582316:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879504/real 1788879507] req@ffffa06188871880 x1875771511632512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879507.582341:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511632512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879507.582357:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879507.582365:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879507.582372:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879507.582426:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879504/real 1788879507] req@ffffa06188873800 x1875771511632640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879507.582453:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511632640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879507.582469:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879507.582475:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879507.582482:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879507.582495:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879504/real 1788879507] req@ffffa06188871c00 x1875771511632128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879507.582519:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511632128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879507.582534:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879507.582540:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879507.582547:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511632128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879507.582559:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879504/real 1788879507] req@ffffa06188870a80 x1875771511632256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879507.582584:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511632256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879507.582601:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879507.582608:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879507.582615:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788879508.221540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879508.221709:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879508.477491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879509.245535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879509.245756:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879509.501497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879509.629528:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879509.629545:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879509.629552:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879509.629572:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4496 00000100:00080000:3.0:1788879509.629596:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879509.629612:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879509.629702:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511632768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879509.629749:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879509.629757:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879509.629761:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879509.629770:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4496 00000100:00080000:3.0:1788879509.629779:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879509.629788:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879509.629807:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879509.629865:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511632768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879509.629870:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511632896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879509.629932:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879509.629939:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879509.629943:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879509.629952:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4496 00000100:00080000:3.0:1788879509.629962:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879509.629971:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879509.629999:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879509.630026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879509.630039:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511633024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879509.630064:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879509.630071:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879509.630075:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879509.630100:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879509/real 1788879509] req@ffffa06188870700 x1875771511632768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879509.630108:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4496 00000100:00080000:3.0:1788879509.630122:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879509.630136:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879509.630148:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879509.630150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511632768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879509.630180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788879509.630181:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511633152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879509.630198:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879509.630217:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511632768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879509.630218:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879509.630236:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879509.630246:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879509.630258:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4496 00000100:00080000:3.0:1788879509.630268:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879509.630275:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879509.630285:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879509.630290:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879509.630314:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511633280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879509.630350:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879509.630382:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879509/real 1788879509] req@ffffa06188873b80 x1875771511632896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879509.630456:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511632896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879509.630472:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879509.630479:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879509.630485:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511632896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879509.630497:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879509.630536:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879509.630571:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879509/real 1788879509] req@ffffa06188872300 x1875771511633024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879509.630596:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511633024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879509.630613:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879509.630619:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879509.630653:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879509.630666:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879509.630708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879509.630731:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879509/real 1788879509] req@ffffa06188872a00 x1875771511633152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879509.630755:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511633152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879509.630769:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879509.630774:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879509.630780:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879509.630814:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879509.630851:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879509.630874:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879509/real 1788879509] req@ffffa06188872d80 x1875771511633280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879509.630898:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511633280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879509.630915:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879509.630922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879509.630928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879510.269552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879510.269749:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879510.525480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879511.293552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879511.293794:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879511.549538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879512.317531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879512.317658:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879512.573534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879513.341540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879513.341695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879513.597547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879514.365442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879514.365584:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879514.621482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879514.749643:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879514.749657:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879514.749663:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879514.749695:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4501 00000100:00080000:1.0:1788879514.749712:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879514.749726:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879514.749786:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511633408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879514.749853:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879514.749858:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879514.749861:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879514.749866:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4501 00000100:00080000:1.0:1788879514.749872:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879514.749878:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879514.749911:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879514.749924:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511633408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879514.749934:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511633536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879514.749951:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879514.749956:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879514.749959:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879514.749963:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4501 00000100:00080000:1.0:1788879514.749968:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879514.749973:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879514.749979:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879514.749994:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511633664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879514.750006:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879514.750011:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879514.750013:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879514.750020:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4501 00000800:00000100:0.0:1788879514.750039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879514.750041:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879514.750052:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879514.750059:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879514.750075:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511633792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879514.750089:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879514.750093:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879514.750095:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879514.750102:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879514/real 1788879514] req@ffffa0618869b100 x1875771511633408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879514.750105:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4501 00000100:00080000:1.0:1788879514.750113:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879514.750150:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879514.750161:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879514.750176:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511633408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879514.750177:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511633920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879514.750258:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879514.750271:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879514.750281:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511633408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879514.750327:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879514.750363:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879514.750381:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879514/real 1788879514] req@ffffa0618869aa00 x1875771511633536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879514.750416:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511633536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879514.750427:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879514.750431:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879514.750435:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879514.750441:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879514.750507:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879514.750520:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879514/real 1788879514] req@ffffa06188699c00 x1875771511633664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879514.750532:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511633664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879514.750547:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879514.750550:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879514.750554:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879514.750564:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879514.750618:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879514.750644:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879514/real 1788879514] req@ffffa0618869b800 x1875771511633792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879514.750663:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511633792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879514.750690:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879514.750694:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879514.750698:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879514.750705:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879514.750732:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879514.750745:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879514/real 1788879514] req@ffffa06188698380 x1875771511633920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879514.750758:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511633920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879514.750768:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879514.750771:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879514.750776:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511633920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879515.389485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879515.389596:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879515.645516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879516.413469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879516.413616:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879516.669501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879517.437489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879517.437596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879517.693441:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879518.461477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879518.461722:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879518.717534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879519.485543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879519.485693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879519.741522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879519.869636:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879519.869654:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879519.869661:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879519.869678:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4506 00000100:00080000:3.0:1788879519.869696:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879519.869710:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879519.869767:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511634048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879519.869815:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879519.869824:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879519.869828:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879519.869837:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4506 00000100:00080000:3.0:1788879519.869846:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879519.869855:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879519.869872:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879519.869878:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511634048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879519.869906:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511634176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879519.869934:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879519.869946:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879519.869950:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879519.869959:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4506 00000100:00080000:3.0:1788879519.869969:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879519.869977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879519.869985:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879519.869996:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879519.870016:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511634304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879519.870032:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879519.870037:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879519.870040:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879519.870045:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4506 00000100:00080000:3.0:1788879519.870050:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879519.870058:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879519.870065:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879519/real 1788879519] req@ffffa06188872680 x1875771511634048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879519.870065:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879519.870084:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511634432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879519.870112:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879519.870118:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879519.870121:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879519.870131:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4506 00000100:00100000:0.0:1788879519.870133:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511634048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879519.870136:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879519.870149:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879519.870163:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879519.870171:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879519.870181:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879519.870187:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511634560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879519.870194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511634048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879519.870243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879519.870283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879519.870301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879519/real 1788879519] req@ffffa06188873100 x1875771511634176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879519.870317:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511634176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879519.870328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879519.870332:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879519.870337:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879519.870345:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879519.870366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879519.870378:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879519/real 1788879519] req@ffffa06188871500 x1875771511634304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879519.870434:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511634304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879519.870451:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879519.870457:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879519.870464:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879519.870477:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879519.870519:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879519.870541:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879519/real 1788879519] req@ffffa06188873480 x1875771511634432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879519.870582:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511634432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879519.870599:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879519.870606:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879519.870612:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879519.870625:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879519.870668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879519.870691:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879519/real 1788879519] req@ffffa06188872d80 x1875771511634560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879519.870708:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511634560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879519.870724:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879519.870731:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879519.870737:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879520.509496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879520.509651:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879520.765531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879521.533558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879521.533705:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879521.789582:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879522.557550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879522.557693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879522.813500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879523.581571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879523.581684:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879523.837448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879524.605476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879524.605590:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879524.861488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879524.989465:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879524.989474:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879524.989478:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879524.989487:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4511 00000100:00080000:3.0:1788879524.989498:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879524.989506:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879524.989561:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511634688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879524.989583:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879524.989585:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879524.989586:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879524.989589:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4511 00000100:00080000:3.0:1788879524.989592:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879524.989595:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879524.989605:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879524.989614:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511634816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879524.989622:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511634688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879524.989625:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879524.989631:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879524.989632:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879524.989635:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4511 00000100:00080000:3.0:1788879524.989638:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879524.989641:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879524.989645:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879524.989654:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511634944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879524.989660:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879524.989663:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879524.989664:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879524.989667:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4511 00000100:00080000:3.0:1788879524.989670:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879524.989672:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879524.989675:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879524.989682:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511635072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879524.989688:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879524.989692:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879524.989693:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879524.989695:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4511 00000100:00080000:3.0:1788879524.989697:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879524.989699:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879524.989702:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879524.989708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879524.989712:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511635200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879524.989757:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879524/real 1788879524] req@ffffa06188872a00 x1875771511634688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879524.989775:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511634688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879524.989784:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879524.989787:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879524.989790:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511634688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879524.989810:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879524.989829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879524.989837:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879524/real 1788879524] req@ffffa06188872300 x1875771511634816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879524.989849:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511634816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879524.989855:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879524.989857:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879524.989859:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879524.989862:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879524.989873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879524.989879:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879524/real 1788879524] req@ffffa06188873b80 x1875771511634944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879524.989886:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511634944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879524.989891:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879524.989893:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879524.989895:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511634944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879524.989898:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879524.989909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879524.989914:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879524/real 1788879524] req@ffffa06188870700 x1875771511635072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879524.989921:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511635072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879524.989931:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879524.989934:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879524.989936:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879524.989939:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879524.989950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879524.989955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879524/real 1788879524] req@ffffa06188870a80 x1875771511635200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879524.989962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511635200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879524.989967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879524.989969:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879524.989971:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879525.629536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879525.629655:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879525.885493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879526.653471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879526.653701:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879526.909557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879527.677550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879527.677648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879527.933565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879528.701538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879528.701649:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879528.957475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879529.725501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879529.725616:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879529.981496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879530.109581:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879530.109595:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879530.109601:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879530.109614:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4516 00000100:00080000:3.0:1788879530.109629:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879530.109652:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879530.109718:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511635328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879530.109762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879530.109767:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879530.109769:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879530.109774:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4516 00000100:00080000:3.0:1788879530.109779:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879530.109784:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879530.109798:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879530.109816:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511635456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879530.109832:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879530.109842:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788879530.109845:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511635328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879530.109847:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879530.109867:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4516 00000100:00080000:3.0:1788879530.109872:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879530.109876:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879530.109883:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879530.109898:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511635584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879530.109908:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879530.109912:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879530.109914:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879530.109918:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4516 00000100:00080000:3.0:1788879530.109925:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879530.109928:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879530.109934:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879530.109945:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511635712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879530.109957:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879530.109970:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788879530.109972:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879530.109974:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879530.109985:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4516 00000100:00080000:3.0:1788879530.109989:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879530.109993:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879530.109999:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879530.110014:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511635840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879530.110062:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879530/real 1788879530] req@ffffa06188871c00 x1875771511635328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879530.110100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511635328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879530.110130:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879530.110136:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879530.110141:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511635328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879530.110174:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879530.110219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879530.110243:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879530/real 1788879530] req@ffffa06188873800 x1875771511635456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879530.110266:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511635456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879530.110277:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879530.110281:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879530.110285:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879530.110292:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879530.110318:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879530.110331:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879530/real 1788879530] req@ffffa06188871880 x1875771511635584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879530.110344:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511635584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879530.110353:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879530.110356:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879530.110360:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879530.110368:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879530.110421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879530.110435:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879530/real 1788879530] req@ffffa06188870380 x1875771511635712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879530.110448:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511635712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879530.110457:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879530.110461:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879530.110464:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879530.110471:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879530.110520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879530.110534:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879530/real 1788879530] req@ffffa06188870a80 x1875771511635840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879530.110547:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511635840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879530.110556:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879530.110559:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879530.110563:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511635840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879530.749581:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879530.749696:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879531.005500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879531.773541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879531.773693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879532.029532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879532.797576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879532.797691:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879533.053590:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879533.821527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879533.821627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879534.077480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879534.845578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879534.845674:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879535.101553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879535.229538:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879535.229551:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879535.229559:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879535.229576:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4521 00000100:00080000:3.0:1788879535.229592:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879535.229603:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879535.229665:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511635968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879535.229706:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879535.229710:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879535.229713:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879535.229718:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4521 00000100:00080000:3.0:1788879535.229722:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879535.229726:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879535.229757:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879535.229774:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511636096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879535.229788:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879535.229793:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879535.229796:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879535.229801:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4521 00000100:00080000:3.0:1788879535.229806:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879535.229819:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879535.229826:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879535.229836:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511635968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879535.229846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511636224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879535.229871:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879535.229875:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879535.229878:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879535.229882:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4521 00000100:00080000:3.0:1788879535.229886:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879535.229891:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879535.229897:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879535.229912:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511636352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879535.229924:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879535.229929:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879535.229931:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879535.229940:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4521 00000800:00000100:0.0:1788879535.229944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879535.229976:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879535.229983:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879535.229990:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879535.230008:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511636480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879535.230034:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879535/real 1788879535] req@ffffa06188870700 x1875771511635968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879535.230060:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511635968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879535.230075:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879535.230079:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879535.230102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511635968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879535.230112:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879535.230141:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879535.230167:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879535/real 1788879535] req@ffffa06188873b80 x1875771511636096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879535.230182:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511636096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879535.230192:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879535.230196:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879535.230201:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879535.230231:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879535.230251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879535.230283:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879535/real 1788879535] req@ffffa06188872300 x1875771511636224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879535.230307:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511636224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879535.230317:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879535.230321:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879535.230325:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879535.230332:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879535.230373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879535.230385:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879535/real 1788879535] req@ffffa06188872a00 x1875771511636352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879535.230429:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511636352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879535.230464:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879535.230471:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879535.230477:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879535.230488:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879535.230526:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879535.230540:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879535/real 1788879535] req@ffffa06188872d80 x1875771511636480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879535.230553:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511636480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879535.230570:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879535.230574:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879535.230578:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879535.869559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879535.869717:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879536.125552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879536.893532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879536.893670:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879537.149559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879537.917555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879537.917730:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879538.173526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879538.941542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879538.941641:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879539.197504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879539.965555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879539.965729:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879540.221531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879540.349530:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879540.349544:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879540.349550:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879540.349567:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4527 00000100:00080000:3.0:1788879540.349585:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879540.349598:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879540.349651:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511636608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879540.349689:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879540.349694:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879540.349697:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879540.349702:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4527 00000100:00080000:3.0:1788879540.349707:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879540.349713:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879540.349727:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879540.349745:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511636736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879540.349756:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511636608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879540.349764:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879540.349775:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879540.349779:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879540.349785:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4527 00000100:00080000:3.0:1788879540.349789:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879540.349793:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879540.349799:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879540.349815:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511636864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879540.349828:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879540.349833:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879540.349836:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879540.349841:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4527 00000100:00080000:3.0:1788879540.349849:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879540.349853:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879540.349859:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879540.349875:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511636992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879540.349890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879540.349896:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879540.349903:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879540.349906:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879540.349913:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4527 00000100:00080000:3.0:1788879540.349917:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879540.349923:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879540.349929:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879540.349945:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511637120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879540.349985:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879540/real 1788879540] req@ffffa06188873480 x1875771511636608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879540.350020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511636608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879540.350036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879540.350041:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879540.350047:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511636608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879540.350080:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879540.350127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879540.350145:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879540/real 1788879540] req@ffffa06188871500 x1875771511636736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879540.350161:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511636736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879540.350180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879540.350184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879540.350189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879540.350196:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879540.350216:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879540.350228:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879540/real 1788879540] req@ffffa06188873100 x1875771511636864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879540.350251:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511636864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879540.350261:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879540.350265:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879540.350269:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879540.350276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879540.350297:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879540.350309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879540/real 1788879540] req@ffffa06188872680 x1875771511636992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879540.350322:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511636992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879540.350332:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879540.350335:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879540.350339:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511636992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879540.350346:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879540.350364:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879540.350375:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879540/real 1788879540] req@ffffa06188872d80 x1875771511637120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879540.350462:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511637120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879540.350479:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879540.350486:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879540.350493:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879540.989526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879540.989654:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879541.245505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879542.013482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879542.013567:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879542.269447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879543.037521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879543.037570:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879543.293519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879544.061562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879544.061737:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879544.317522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879545.085553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879545.085736:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879545.341517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879545.469460:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879545.469468:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879545.469472:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879545.469482:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4532 00000100:00080000:3.0:1788879545.469492:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879545.469499:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879545.469534:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511637248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879545.469557:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879545.469560:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879545.469561:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879545.469565:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4532 00000100:00080000:3.0:1788879545.469568:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879545.469571:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879545.469580:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879545.469589:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511637376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879545.469614:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511637248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879545.469619:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879545.469629:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879545.469631:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879545.469636:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4532 00000100:00080000:3.0:1788879545.469639:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879545.469643:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879545.469647:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879545.469661:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511637504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879545.469670:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879545.469674:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879545.469676:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879545.469679:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4532 00000100:00080000:3.0:1788879545.469683:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879545.469687:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879545.469691:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879545.469702:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511637632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879545.469709:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879545.469713:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879545.469715:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879545.469723:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879545.469726:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4532 00000100:00080000:3.0:1788879545.469732:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879545.469736:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879545.469741:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879545.469753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511637760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879545.469787:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879545/real 1788879545] req@ffffa06188872a00 x1875771511637248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879545.469808:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511637248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879545.469820:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879545.469823:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879545.469827:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511637248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879545.469867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879545.469887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879545.469896:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879545/real 1788879545] req@ffffa06188872300 x1875771511637376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879545.469905:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511637376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879545.469912:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879545.469914:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879545.469917:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879545.469921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879545.469939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879545.469948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879545/real 1788879545] req@ffffa06188873b80 x1875771511637504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879545.469956:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511637504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879545.469962:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879545.469964:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879545.469967:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879545.469971:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879545.469983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879545.469990:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879545/real 1788879545] req@ffffa06188870700 x1875771511637632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879545.470009:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511637632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879545.470018:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879545.470022:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879545.470026:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879545.470033:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879545.470057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879545.470068:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879545/real 1788879545] req@ffffa06188870a80 x1875771511637760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879545.470076:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511637760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879545.470083:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879545.470085:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879545.470087:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511637760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879546.109439:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879546.109498:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879546.365454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879547.133473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879547.133515:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879547.389497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879548.157587:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879548.157706:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879548.413467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879549.181463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879549.181504:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879549.437470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879550.205529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879550.205592:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879550.461479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879550.589483:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879550.589495:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879550.589500:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879550.589515:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4537 00000100:00080000:3.0:1788879550.589529:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879550.589541:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879550.589587:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511637888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879550.589629:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879550.589636:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879550.589639:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879550.589643:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4537 00000100:00080000:3.0:1788879550.589647:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879550.589651:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879550.589666:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879550.589694:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511637888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879550.589694:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511638016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879550.589724:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879550.589729:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879550.589733:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879550.589740:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4537 00000100:00080000:3.0:1788879550.589746:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879550.589751:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879550.589759:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879550.589774:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511638144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879550.589786:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879550.589791:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879550.589793:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879550.589798:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879550.589804:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4537 00000100:00080000:3.0:1788879550.589810:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879550.589814:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879550.589822:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879550.589834:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511638272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879550.589842:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879550/real 1788879550] req@ffffa06188870380 x1875771511637888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879550.589853:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879550.589865:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879550.589870:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879550.589881:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4537 00000100:00080000:3.0:1788879550.589890:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879550.589899:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879550.589905:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879550.589914:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511637888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879550.589922:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511638400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879550.589941:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879550.589950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879550.589955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511637888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879550.589984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879550.590007:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879550.590020:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879550/real 1788879550] req@ffffa06188871880 x1875771511638016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879550.590033:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511638016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879550.590042:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879550.590045:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879550.590048:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879550.590059:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879550.590075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879550.590084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879550/real 1788879550] req@ffffa06188873800 x1875771511638144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879550.590095:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511638144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879550.590109:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879550.590113:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879550.590116:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879550.590122:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879550.590146:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879550.590162:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879550/real 1788879550] req@ffffa06188871c00 x1875771511638272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879550.590179:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511638272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879550.590210:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879550.590216:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879550.590233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879550.590247:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879550.590283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879550.590303:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879550/real 1788879550] req@ffffa06188870a80 x1875771511638400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879550.590323:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511638400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879550.590377:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879550.590412:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879550.590419:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879551.229524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879551.229591:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879551.485474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879552.253493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879552.253566:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879552.509572:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879553.277560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879553.277617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879553.533497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879554.301521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879554.301624:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879554.557532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879555.325506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879555.325629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879555.581562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879555.709626:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879555.709640:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879555.709646:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879555.709662:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4542 00000100:00080000:1.0:1788879555.709680:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879555.709691:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879555.709746:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511638528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879555.709788:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879555.709792:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879555.709795:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879555.709800:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4542 00000100:00080000:1.0:1788879555.709806:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879555.709812:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879555.709826:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879555.709844:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511638656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879555.709865:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511638528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879555.709866:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879555.709888:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879555.709893:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879555.709901:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4542 00000100:00080000:1.0:1788879555.709904:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879555.709911:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879555.709918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879555.709939:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511638784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879555.709951:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879555.709956:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879555.709958:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879555.709966:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4542 00000100:00080000:1.0:1788879555.709986:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879555.709993:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879555.709999:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879555.710010:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879555.710027:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511638912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879555.710044:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879555.710049:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879555.710052:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879555.710060:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4542 00000100:00080000:1.0:1788879555.710068:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879555.710074:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879555.710082:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879555.710084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879555/real 1788879555] req@ffffa06188699f80 x1875771511638528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879555.710116:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511639040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879555.710128:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511638528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879555.710167:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879555.710176:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879555.710183:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511638528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879555.710224:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879555.710253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879555.710269:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879555/real 1788879555] req@ffffa06188698000 x1875771511638656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879555.710285:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511638656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879555.710324:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879555.710329:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879555.710334:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879555.710344:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879555.710370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879555.710417:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879555/real 1788879555] req@ffffa0618869a680 x1875771511638784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879555.710433:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511638784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879555.710444:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879555.710448:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879555.710452:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879555.710460:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879555.710484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879555.710498:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879555/real 1788879555] req@ffffa0618869bb80 x1875771511638912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879555.710512:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511638912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879555.710523:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879555.710527:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879555.710531:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511638912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879555.710538:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879555.710557:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879555.710568:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879555/real 1788879555] req@ffffa0618869ad80 x1875771511639040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879555.710583:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511639040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879555.710593:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879555.710596:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879555.710600:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879556.349529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879556.349683:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879556.605530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879557.373485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879557.373600:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879557.629448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879558.397516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879558.397631:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879558.653487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879559.421537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879559.421612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879559.677446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879560.445530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879560.445621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879560.701476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879560.829549:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879560.829562:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879560.829568:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879560.829582:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4547 00000100:00080000:2.0:1788879560.829596:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879560.829619:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879560.829669:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511639168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879560.829717:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879560.829720:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879560.829721:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879560.829725:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4547 00000100:00080000:2.0:1788879560.829729:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879560.829733:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879560.829743:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879560.829773:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511639168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879560.829781:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511639296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879560.829800:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879560.829804:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879560.829806:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879560.829827:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4547 00000100:00080000:2.0:1788879560.829830:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879560.829832:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879560.829836:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879560.829847:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511639424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879560.829854:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879560.829856:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879560.829857:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879560.829859:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4547 00000100:00080000:2.0:1788879560.829862:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879560.829864:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879560.829867:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879560.829875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879560.829877:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511639552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879560.829885:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879560.829887:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879560.829889:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879560.829892:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4547 00000100:00080000:2.0:1788879560.829896:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879560.829900:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879560.829906:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879560.829912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879560/real 1788879560] req@ffffa061888aca80 x1875771511639168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879560.829933:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511639680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879560.829945:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511639168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879560.829956:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879560.829959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879560.829964:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511639168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879560.829984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879560.830004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879560.830016:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879560/real 1788879560] req@ffffa061888af480 x1875771511639296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879560.830031:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511639296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879560.830040:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879560.830044:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879560.830048:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879560.830054:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879560.830105:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879560.830131:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879560/real 1788879560] req@ffffa061888adf80 x1875771511639424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879560.830164:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511639424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879560.830182:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879560.830186:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879560.830189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879560.830195:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879560.830214:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879560.830223:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879560/real 1788879560] req@ffffa061888ac000 x1875771511639552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879560.830252:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511639552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879560.830326:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879560.830328:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879560.830331:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879560.830335:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879560.830349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879560.830356:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879560/real 1788879560] req@ffffa061888ad500 x1875771511639680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879560.830380:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511639680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879560.830386:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879560.830389:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879560.830391:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879561.469525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879561.469606:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879561.725537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879562.493533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879562.493599:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879562.749522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879563.517482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879563.517603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879563.773490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879564.541557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879564.541698:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879564.797530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879565.565558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879565.565655:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879565.821496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879565.949480:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879565.949492:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879565.949497:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879565.949510:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4552 00000100:00080000:2.0:1788879565.949522:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879565.949531:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879565.949573:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511639808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879565.949601:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879565.949605:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879565.949607:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879565.949611:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4552 00000100:00080000:2.0:1788879565.949615:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879565.949620:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879565.949631:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879565.949644:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511639808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879565.949647:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511639936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879565.949663:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879565.949667:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879565.949669:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879565.949674:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4552 00000100:00080000:2.0:1788879565.949677:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879565.949681:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879565.949686:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879565.949697:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511640064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879565.949724:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879565.949728:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879565.949731:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879565.949736:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4552 00000100:00080000:2.0:1788879565.949742:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879565.949746:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879565.949753:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879565.949757:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879565.949775:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511640192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879565.949791:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879565.949796:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879565.949800:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879565.949806:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4552 00000100:00080000:2.0:1788879565.949812:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879565.949817:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879565.949823:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879565/real 1788879565] req@ffffa061888af100 x1875771511639808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879565.949827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879565.949853:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511640320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879565.949882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511639808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879565.949901:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879565.949907:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879565.949912:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511639808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879565.949962:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879565.949985:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879565.949997:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879565/real 1788879565] req@ffffa061888ae300 x1875771511639936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879565.950009:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511639936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879565.950017:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879565.950020:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879565.950023:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511639936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879565.950028:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879565.950042:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879565.950071:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879565/real 1788879565] req@ffffa061888aea00 x1875771511640064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879565.950085:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511640064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879565.950095:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879565.950098:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879565.950102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879565.950110:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879565.950131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879565.950142:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879565/real 1788879565] req@ffffa061888afb80 x1875771511640192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879565.950156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511640192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879565.950166:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879565.950169:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879565.950173:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879565.950179:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879565.950197:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879565.950208:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879565/real 1788879565] req@ffffa061888adc00 x1875771511640320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879565.950270:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511640320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879565.950282:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879565.950287:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879565.950292:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879566.589454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879566.589508:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879566.845533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879567.613485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879567.613547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879567.869463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879568.637459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879568.637530:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879568.893499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879569.661506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879569.661680:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879569.917472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879570.685489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879570.685585:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879570.941511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879571.069487:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879571.069498:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879571.069502:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879571.069513:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4557 00000100:00080000:3.0:1788879571.069524:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879571.069533:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879571.069571:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511640448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879571.069596:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879571.069600:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879571.069602:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879571.069607:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4557 00000100:00080000:3.0:1788879571.069611:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879571.069616:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879571.069627:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879571.069637:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511640448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879571.069641:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511640576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879571.069653:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879571.069656:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879571.069658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879571.069662:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4557 00000100:00080000:3.0:1788879571.069665:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879571.069667:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879571.069670:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879571.069680:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511640704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879571.069687:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879571.069690:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879571.069692:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879571.069695:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4557 00000100:00080000:3.0:1788879571.069699:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879571.069701:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879571.069705:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879571.069714:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511640832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879571.069726:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879571.069730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879571.069732:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879571.069735:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4557 00000100:00080000:3.0:1788879571.069738:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879571.069741:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879571.069746:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879571.069751:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879571.069760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511640960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879571.069786:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879571.069807:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879571.069833:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879571.709520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879571.709603:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879571.965482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879572.733538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879572.733601:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879572.989525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879573.757498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879573.757645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879574.013536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879574.141881:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879574.141955:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879574.141983:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879574.142050:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879574.142072:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879574.142102:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879574.142125:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879574.142240:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879571/real 1788879574] req@ffffa06188872d80 x1875771511640960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879574.142306:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511640960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879574.142327:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879574.142336:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879574.142343:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879574.142359:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879571/real 1788879574] req@ffffa06188873b80 x1875771511640576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879574.142379:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511640576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879574.142435:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879574.142448:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879574.142457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879574.142472:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879571/real 1788879574] req@ffffa06188872300 x1875771511640704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879574.142495:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511640704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879574.142509:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879574.142515:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879574.142521:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879574.142533:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879571/real 1788879574] req@ffffa06188872a00 x1875771511640832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879574.142550:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511640832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879574.142562:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879574.142567:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879574.142572:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511640832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879574.142584:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879571/real 1788879574] req@ffffa06188870700 x1875771511640448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879574.142604:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511640448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879574.142617:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879574.142623:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879574.142629:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511640448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879574.781529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879574.781633:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879575.037482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879575.805517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879575.805586:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879576.061528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879576.189559:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879576.189571:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879576.189576:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879576.189591:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4562 00000100:00080000:1.0:1788879576.189604:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879576.189613:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879576.189658:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511641088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879576.189690:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879576.189694:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879576.189696:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879576.189701:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4562 00000100:00080000:1.0:1788879576.189705:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879576.189708:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879576.189720:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879576.189734:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511641216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879576.189745:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511641088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879576.189750:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879576.189760:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879576.189764:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879576.189768:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4562 00000100:00080000:1.0:1788879576.189771:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879576.189774:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879576.189779:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879576.189793:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511641344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879576.189804:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879576.189808:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879576.189811:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879576.189814:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4562 00000100:00080000:1.0:1788879576.189840:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879576.189844:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879576.189848:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879576.189860:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879576.189862:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511641472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879576.189876:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879576.189880:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879576.189882:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879576.189886:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4562 00000100:00080000:1.0:1788879576.189895:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879576.189901:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879576.189909:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879576.189922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879576/real 1788879576] req@ffffa0618869ad80 x1875771511641088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879576.189931:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511641600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879576.189963:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511641088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879576.189983:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879576.189992:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879576.190000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511641088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879576.190037:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879576.190097:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879576.190119:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879576/real 1788879576] req@ffffa0618869bb80 x1875771511641216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879576.190142:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511641216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879576.190177:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879576.190184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879576.190191:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879576.190205:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879576.190240:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879576.190260:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879576/real 1788879576] req@ffffa0618869a680 x1875771511641344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879576.190282:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511641344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879576.190296:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879576.190302:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879576.190307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879576.190318:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879576.190353:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879576.190372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879576/real 1788879576] req@ffffa06188698000 x1875771511641472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879576.190433:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511641472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879576.190448:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879576.190454:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879576.190459:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879576.190469:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879576.190501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879576.190520:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879576/real 1788879576] req@ffffa06188699f80 x1875771511641600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879576.190541:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511641600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879576.190554:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879576.190558:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879576.190563:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879576.829463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879576.829552:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879577.085476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879577.853540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879577.853606:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879578.109637:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879578.877525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879578.877685:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879579.133548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879579.901525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879579.901636:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879580.157540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879580.925528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879580.925657:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879581.181524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879581.309547:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879581.309562:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879581.309569:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879581.309588:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4568 00000100:00080000:3.0:1788879581.309607:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879581.309619:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879581.309682:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511641728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879581.309723:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879581.309728:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879581.309732:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879581.309737:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4568 00000100:00080000:3.0:1788879581.309742:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879581.309746:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879581.309764:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879581.309782:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511641856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879581.309795:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511641728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879581.309804:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879581.309812:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879581.309815:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879581.309821:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4568 00000100:00080000:3.0:1788879581.309825:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879581.309829:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879581.309835:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879581.309852:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511641984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879581.309866:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879581.309871:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879581.309873:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879581.309878:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4568 00000100:00080000:3.0:1788879581.309883:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879581.309888:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879581.309893:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879581.309907:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511642112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879581.309918:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879581.309923:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879581.309929:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879581.309932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879581.309940:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4568 00000100:00080000:3.0:1788879581.309946:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879581.309952:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879581.309961:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879581.309989:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511642240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879581.310004:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879581/real 1788879581] req@ffffa06188872680 x1875771511641728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879581.310043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511641728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879581.310066:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879581.310076:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879581.310085:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511641728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879581.310155:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879581.310202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879581.310222:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879581/real 1788879581] req@ffffa06188873100 x1875771511641856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879581.310258:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511641856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879581.310271:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879581.310275:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879581.310280:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879581.310288:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879581.310314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879581.310328:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879581/real 1788879581] req@ffffa06188871500 x1875771511641984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879581.310343:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511641984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879581.310353:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879581.310356:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879581.310360:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511641984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879581.310367:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879581.310430:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879581.310452:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879581/real 1788879581] req@ffffa06188873480 x1875771511642112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879581.310478:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511642112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879581.310512:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879581.310518:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879581.310525:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879581.310536:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879581.310568:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879581.310584:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879581/real 1788879581] req@ffffa06188870700 x1875771511642240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879581.310598:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511642240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879581.310609:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879581.310613:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879581.310617:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879581.949469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879581.949538:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879582.205459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879582.973446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879582.973534:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879583.229467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879583.997465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879583.997540:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879584.253510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879585.021466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879585.021542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879585.277457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879586.045541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879586.045633:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879586.301463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879586.429530:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879586.429543:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879586.429549:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879586.429566:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4573 00000100:00080000:3.0:1788879586.429581:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879586.429615:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879586.429662:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511642368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879586.429694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879586.429699:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879586.429701:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879586.429706:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4573 00000100:00080000:3.0:1788879586.429709:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879586.429714:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879586.429726:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879586.429740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511642496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879586.429759:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511642368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879586.429763:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879586.429775:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879586.429778:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879586.429783:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4573 00000100:00080000:3.0:1788879586.429786:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879586.429789:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879586.429794:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879586.429807:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511642624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879586.429818:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879586.429822:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879586.429825:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879586.429829:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4573 00000100:00080000:3.0:1788879586.429832:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879586.429835:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879586.429840:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879586.429852:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511642752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879586.429864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879586.429871:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879586.429877:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879586.429879:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879586.429884:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4573 00000100:00080000:3.0:1788879586.429890:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879586.429913:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879586.429920:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879586.429953:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879586/real 1788879586] req@ffffa06188872a00 x1875771511642368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879586.429953:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511642880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879586.430023:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511642368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879586.430042:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879586.430048:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879586.430053:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511642368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879586.430143:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879586.430171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879586.430206:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879586/real 1788879586] req@ffffa06188872300 x1875771511642496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879586.430220:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511642496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879586.430231:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879586.430235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879586.430239:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879586.430246:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879586.430267:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879586.430278:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879586/real 1788879586] req@ffffa06188873b80 x1875771511642624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879586.430290:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511642624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879586.430300:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879586.430303:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879586.430306:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879586.430313:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879586.430331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879586.430341:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879586/real 1788879586] req@ffffa06188872d80 x1875771511642752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879586.430354:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511642752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879586.430363:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879586.430366:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879586.430369:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879586.430375:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879586.430402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879586.430412:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879586/real 1788879586] req@ffffa06188870a80 x1875771511642880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879586.430458:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511642880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879586.430467:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879586.430471:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879586.430474:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511642880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879587.069534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879587.069672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879587.325551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879588.093572:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879588.093651:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879588.349519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879589.117510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879589.117619:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879589.373629:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879590.141507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879590.141666:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879590.397596:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879591.165505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879591.165531:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879591.421576:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879591.549645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879591.549658:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879591.549664:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879591.549678:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4578 00000100:00080000:2.0:1788879591.549694:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879591.549705:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879591.549759:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511643008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879591.549799:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879591.549805:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879591.549808:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879591.549830:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4578 00000100:00080000:2.0:1788879591.549835:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879591.549843:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879591.549857:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879591.549872:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511643008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879591.549897:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511643136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879591.549915:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879591.549919:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879591.549922:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879591.549927:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4578 00000100:00080000:2.0:1788879591.549931:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879591.549936:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879591.549943:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879591.549960:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511643264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879591.549973:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879591.549978:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879591.549981:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879591.549985:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4578 00000100:00080000:2.0:1788879591.549990:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879591.549998:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879591.550004:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879591.550013:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879591.550028:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511643392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879591.550068:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879591.550074:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879591.550077:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879591.550084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879591/real 1788879591] req@ffffa061888aed80 x1875771511643008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879591.550088:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4578 00000100:00080000:2.0:1788879591.550118:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879591.550128:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879591.550137:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879591.550147:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511643008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879591.550157:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511643520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879591.550179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879591.550188:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879591.550195:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511643008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879591.550230:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879591.550282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879591.550299:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879591/real 1788879591] req@ffffa061888ac700 x1875771511643136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879591.550315:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511643136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879591.550326:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879591.550330:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879591.550335:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879591.550342:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879591.550360:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879591.550372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879591/real 1788879591] req@ffffa061888ac380 x1875771511643264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879591.550387:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511643264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879591.550396:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879591.550400:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879591.550404:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879591.550410:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879591.550503:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879591.550518:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879591/real 1788879591] req@ffffa061888ad880 x1875771511643392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879591.550532:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511643392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879591.550542:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879591.550562:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879591.550567:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879591.550575:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879591.550597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879591.550609:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879591/real 1788879591] req@ffffa061888ae680 x1875771511643520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879591.550623:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511643520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879591.550641:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879591.550645:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879591.550649:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879592.189516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879592.189654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879592.445517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879593.213463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879593.213556:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879593.469487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879594.237454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879594.237515:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879594.493460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879595.261450:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879595.261540:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879595.517544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879596.285487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879596.285522:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879596.541472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879596.669530:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879596.669539:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879596.669543:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879596.669553:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4583 00000100:00080000:3.0:1788879596.669563:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879596.669570:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879596.669607:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511643648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879596.669644:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879596.669647:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879596.669648:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879596.669651:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4583 00000100:00080000:3.0:1788879596.669653:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879596.669656:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879596.669664:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879596.669674:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511643776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879596.669681:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879596.669685:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879596.669686:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879596.669689:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4583 00000100:00080000:3.0:1788879596.669692:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879596.669697:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879596.669698:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511643648:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879596.669701:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879596.669712:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511643904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879596.669726:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879596.669729:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879596.669730:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879596.669732:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4583 00000100:00080000:3.0:1788879596.669734:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879596.669736:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879596.669739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879596.669747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511644032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879596.669753:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879596.669756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879596.669757:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879596.669760:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4583 00000100:00080000:3.0:1788879596.669763:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879596.669765:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879596.669768:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879596.669776:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511644160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879596.669791:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879596.669826:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879596/real 1788879596] req@ffffa06188871c00 x1875771511643648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879596.669842:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511643648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879596.669850:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879596.669853:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879596.669856:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511643648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879596.669862:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879596.669879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879596.669886:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879596/real 1788879596] req@ffffa06188873800 x1875771511643776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879596.669894:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511643776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879596.669900:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879596.669902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879596.669904:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879596.669932:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879596.669943:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879596.669950:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879596/real 1788879596] req@ffffa06188871880 x1875771511643904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879596.669957:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511643904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879596.669963:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879596.669965:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879596.669967:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511643904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879596.669970:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879596.669979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879596.669985:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879596/real 1788879596] req@ffffa06188870380 x1875771511644032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879596.669992:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511644032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879596.669997:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879596.669999:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879596.670001:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879596.670005:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879596.670035:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879596.670053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879596/real 1788879596] req@ffffa06188870a80 x1875771511644160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879596.670060:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511644160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879596.670066:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879596.670068:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879596.670070:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879597.309483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879597.309590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879597.565479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879598.333497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879598.333645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879598.589524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879599.357483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879599.357715:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879599.613489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879600.381519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879600.381592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879600.637460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879601.405529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879601.405586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879601.661502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879601.789560:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879601.789577:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879601.789585:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879601.789602:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4588 00000100:00080000:3.0:1788879601.789621:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879601.789632:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879601.789685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511644288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879601.789725:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879601.789730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879601.789733:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879601.789738:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4588 00000100:00080000:3.0:1788879601.789743:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879601.789748:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879601.789762:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879601.789783:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511644288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879601.789784:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511644416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879601.789813:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879601.789817:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879601.789819:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879601.789824:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4588 00000100:00080000:3.0:1788879601.789829:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879601.789835:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879601.789841:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879601.789857:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511644544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879601.789870:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879601.789874:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879601.789877:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879601.789882:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4588 00000100:00080000:3.0:1788879601.789886:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879601.789891:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879601.789900:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788879601.789902:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879601.789921:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511644672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879601.789934:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879601.789939:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879601.789942:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879601.789947:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4588 00000100:00080000:3.0:1788879601.789952:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879601.789958:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879601/real 1788879601] req@ffffa06188872d80 x1875771511644288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879601.789963:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879601.790020:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879601.790046:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511644800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879601.790055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511644288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879601.790132:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879601.790140:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879601.790148:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511644288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879601.790203:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879601.790248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879601.790272:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879601/real 1788879601] req@ffffa06188873b80 x1875771511644416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879601.790296:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511644416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879601.790307:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879601.790311:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879601.790316:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879601.790323:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879601.790350:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879601.790363:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879601/real 1788879601] req@ffffa06188872300 x1875771511644544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879601.790376:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511644544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879601.790386:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879601.790389:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879601.790393:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879601.790429:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879601.790452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879601.790466:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879601/real 1788879601] req@ffffa06188872a00 x1875771511644672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879601.790480:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511644672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879601.790490:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879601.790494:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879601.790497:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879601.790503:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879601.790520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879601.790531:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879601/real 1788879601] req@ffffa06188870700 x1875771511644800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879601.790544:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511644800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879601.790553:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879601.790574:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879601.790577:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511644800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879602.429484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879602.429580:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879602.685512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879603.453494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879603.453580:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879603.709486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879604.477548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879604.477669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879604.733514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879605.501511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879605.501686:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879605.757473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879606.525494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879606.525560:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879606.781483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879606.909576:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879606.909603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879606.909611:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879606.909629:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4593 00000100:00080000:1.0:1788879606.909647:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879606.909661:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879606.909730:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511644928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879606.909772:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879606.909779:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879606.909794:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879606.909802:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4593 00000100:00080000:1.0:1788879606.909810:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879606.909819:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879606.909836:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511644928:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879606.909837:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879606.909885:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511645056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879606.909906:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879606.909912:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879606.909915:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879606.909922:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4593 00000100:00080000:1.0:1788879606.909930:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879606.909968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879606.909976:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879606.909990:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879606.910015:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511645184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879606.910025:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879606/real 1788879606] req@ffffa06188698380 x1875771511644928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879606.910054:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879606.910063:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788879606.910069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511644928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879606.910071:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879606.910090:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4593 00000100:00080000:0.0:1788879606.910094:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879606.910102:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879606.910104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879606.910116:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511644928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879606.910121:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879606.910136:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879606.910155:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879606.910164:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511645312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879606.910188:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879606.910194:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879606.910198:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879606.910207:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879606.910221:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4593 00000100:00080000:1.0:1788879606.910233:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879606.910245:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879606/real 1788879606] req@ffffa0618869b800 x1875771511645056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879606.910246:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879606.910263:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879606.910286:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511645056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879606.910306:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511645440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879606.910310:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879606.910328:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879606.910340:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879606.910351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879606.910415:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879606.910432:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879606/real 1788879606] req@ffffa06188699c00 x1875771511645184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879606.910446:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511645184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879606.910462:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879606.910466:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879606.910470:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879606.910488:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879606.910511:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879606.910530:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879606/real 1788879606] req@ffffa0618869aa00 x1875771511645312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879606.910542:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511645312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879606.910552:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879606.910555:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879606.910558:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879606.910565:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879606.910593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879606.910612:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879606/real 1788879606] req@ffffa0618869b100 x1875771511645440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879606.910633:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511645440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879606.910649:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879606.910655:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879606.910661:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879607.549521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879607.549669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879607.805548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879608.573509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879608.573610:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879608.829488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879609.597475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879609.597665:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879609.853498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879610.621461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879610.621523:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879610.877520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879611.645544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879611.645651:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879611.901487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879612.029555:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879612.029568:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879612.029574:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879612.029590:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4598 00000100:00080000:1.0:1788879612.029606:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879612.029617:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879612.029672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511645568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879612.029706:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879612.029711:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879612.029715:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879612.029721:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4598 00000100:00080000:1.0:1788879612.029726:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879612.029731:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879612.029746:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879612.029770:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511645568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879612.029771:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511645696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879612.029798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879612.029803:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879612.029805:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879612.029810:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4598 00000100:00080000:1.0:1788879612.029815:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879612.029819:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879612.029827:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879612.029844:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511645824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879612.029856:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879612.029861:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879612.029864:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879612.029869:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4598 00000100:00080000:1.0:1788879612.029874:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879612.029880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879612.029888:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879612.029896:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879612.029941:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511645952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879612.029965:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879612.029973:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879612/real 1788879612] req@ffffa0618869b100 x1875771511645568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879612.029974:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879612.029982:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879612.029998:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4598 00000100:00080000:1.0:1788879612.030006:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879612.030012:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879612.030019:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879612.030021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511645568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879612.030043:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511646080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879612.030044:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879612.030058:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879612.030080:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511645568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879612.030129:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879612.030159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879612.030176:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879612/real 1788879612] req@ffffa0618869aa00 x1875771511645696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879612.030192:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511645696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879612.030203:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879612.030208:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879612.030213:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879612.030221:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879612.030240:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879612.030252:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879612/real 1788879612] req@ffffa06188699c00 x1875771511645824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879612.030266:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511645824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879612.030276:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879612.030279:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879612.030283:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879612.030290:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879612.030309:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879612.030320:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879612/real 1788879612] req@ffffa0618869b800 x1875771511645952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879612.030334:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511645952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879612.030344:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879612.030347:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879612.030351:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511645952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879612.030358:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879612.030377:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879612.030424:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879612/real 1788879612] req@ffffa06188698380 x1875771511646080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879612.030439:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511646080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879612.030450:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879612.030453:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879612.030457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879612.669531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879612.669629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879612.925499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879613.693561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879613.693694:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879613.949489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879614.717531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879614.717607:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879614.973562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879615.741514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879615.741695:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879615.997491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:0.0:1788879616.765629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788879616.765686:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788879617.021580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879617.149594:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879617.149609:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879617.149616:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879617.149632:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4603 00000100:00080000:3.0:1788879617.149649:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879617.149660:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879617.149715:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511646208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879617.149755:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879617.149763:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879617.149766:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879617.149775:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4603 00000100:00080000:3.0:1788879617.149784:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879617.149793:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879617.149808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511646208:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879617.149811:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879617.149846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511646336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879617.149906:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879617.149914:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879617.149918:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879617.149926:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4603 00000800:00000100:0.0:1788879617.149936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879617.149939:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879617.149949:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879617.149959:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879617.149979:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511646464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879617.149992:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879617.149997:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879617/real 1788879617] req@ffffa06188873480 x1875771511646208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879617.149998:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879617.150004:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879617.150016:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4603 00000100:00080000:3.0:1788879617.150026:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879617.150033:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879617.150040:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879617.150042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511646208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879617.150057:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511646592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879617.150061:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879617.150075:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788879617.150082:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879617.150087:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511646208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879617.150089:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879617.150097:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879617.150104:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4603 00000100:00080000:3.0:1788879617.150108:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879617.150113:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879617.150117:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879617.150132:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879617.150136:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511646720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879617.150173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879617.150193:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879617/real 1788879617] req@ffffa06188871500 x1875771511646336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879617.150208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511646336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879617.150218:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879617.150222:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879617.150226:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879617.150251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879617.150274:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879617.150286:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879617/real 1788879617] req@ffffa06188873100 x1875771511646464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879617.150300:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511646464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879617.150309:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879617.150312:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879617.150316:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879617.150323:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879617.150359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879617.150370:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879617/real 1788879617] req@ffffa06188872680 x1875771511646592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879617.150383:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511646592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879617.150392:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879617.150396:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879617.150400:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879617.150445:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879617.150467:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879617.150479:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879617/real 1788879617] req@ffffa06188873480 x1875771511646720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879617.150492:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511646720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879617.150502:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879617.150505:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879617.150509:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879617.789531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879617.789612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879618.045535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879618.813514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879618.813656:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879619.069547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879619.837545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879619.837637:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879620.093590:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879620.861483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879620.861582:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879621.117620:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879621.885455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879621.885519:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879622.141534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879622.269496:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879622.269504:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879622.269508:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879622.269518:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4608 00000100:00080000:3.0:1788879622.269529:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879622.269536:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879622.269573:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511646848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879622.269596:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879622.269599:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879622.269600:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879622.269603:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4608 00000100:00080000:3.0:1788879622.269606:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879622.269609:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879622.269619:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879622.269631:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511646976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879622.269632:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511646848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879622.269646:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879622.269648:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879622.269650:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879622.269652:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4608 00000100:00080000:3.0:1788879622.269654:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879622.269656:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879622.269660:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879622.269667:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511647104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879622.269678:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879622.269681:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879622.269682:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879622.269685:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4608 00000100:00080000:3.0:1788879622.269687:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879622.269689:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879622.269692:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879622.269701:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511647232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879622.269708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879622.269710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879622.269713:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879622.269714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879622.269717:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4608 00000100:00080000:3.0:1788879622.269720:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879622.269723:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879622.269726:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879622.269733:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511647360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879622.269747:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879622/real 1788879622] req@ffffa06188870700 x1875771511646848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879622.269764:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511646848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879622.269774:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879622.269777:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879622.269780:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511646848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879622.269799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879622.269813:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879622.269821:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879622/real 1788879622] req@ffffa06188872a00 x1875771511646976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879622.269844:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511646976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879622.269851:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879622.269853:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879622.269855:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511646976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879622.269859:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879622.269873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879622.269880:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879622/real 1788879622] req@ffffa06188872300 x1875771511647104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879622.269887:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511647104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879622.269892:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879622.269894:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879622.269895:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879622.269900:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879622.269910:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879622.269916:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879622/real 1788879622] req@ffffa06188873b80 x1875771511647232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879622.269923:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511647232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879622.269929:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879622.269930:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879622.269932:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879622.269935:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879622.269944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879622.269950:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879622/real 1788879622] req@ffffa06188872d80 x1875771511647360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879622.269957:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511647360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879622.269962:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879622.269963:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879622.269965:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879622.909548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879622.909699:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879623.165486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879623.933577:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879623.933669:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879624.189460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879624.957455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879624.957496:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879625.213472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879625.981583:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879625.981712:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879626.237574:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879627.005483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879627.005596:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879627.261563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879627.389525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879627.389538:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879627.389545:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879627.389560:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4614 00000100:00080000:3.0:1788879627.389578:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879627.389590:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879627.389641:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511647488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879627.389680:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879627.389685:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879627.389687:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879627.389693:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4614 00000100:00080000:3.0:1788879627.389697:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879627.389703:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879627.389718:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879627.389742:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511647616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879627.389744:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511647488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879627.389772:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879627.389776:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879627.389779:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879627.389784:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4614 00000100:00080000:3.0:1788879627.389789:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879627.389830:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879627.389842:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879627.389868:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511647744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879627.389889:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788879627.389901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879627.389905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879627.389912:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879627.389943:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4614 00000100:00080000:3.0:1788879627.389960:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879627.389970:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879627/real 1788879627] req@ffffa06188870a80 x1875771511647488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879627.389970:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879627.389985:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879627.390018:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511647872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879627.390023:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511647488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879627.390076:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879627.390080:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879627.390086:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788879627.390096:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788879627.390097:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879627.390117:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511647488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879627.390117:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4614 00000100:00080000:3.0:1788879627.390130:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879627.390139:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879627.390147:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879627.390167:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511648000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879627.390172:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879627.390228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879627.390249:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879627/real 1788879627] req@ffffa06188870380 x1875771511647616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879627.390266:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511647616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879627.390278:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879627.390282:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879627.390287:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879627.390294:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879627.390315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879627.390327:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879627/real 1788879627] req@ffffa06188871880 x1875771511647744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879627.390342:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511647744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879627.390352:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879627.390356:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879627.390359:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879627.390367:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879627.390424:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879627.390439:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879627/real 1788879627] req@ffffa06188873800 x1875771511647872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879627.390454:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511647872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879627.390465:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879627.390469:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879627.390473:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511647872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879627.390493:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879627.390515:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879627.390527:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879627/real 1788879627] req@ffffa06188871c00 x1875771511648000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879627.390541:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511648000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879627.390559:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879627.390563:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879627.390567:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879628.029539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879628.029667:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879628.285557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879629.053549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879629.053638:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879629.309519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879630.077502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879630.077693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879630.333534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879631.101535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879631.101622:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879631.357482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879632.125501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879632.125639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879632.381522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879632.509572:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879632.509584:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879632.509590:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879632.509604:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4619 00000100:00080000:1.0:1788879632.509618:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879632.509628:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879632.509684:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511648128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879632.509713:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879632.509717:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879632.509719:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879632.509724:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4619 00000100:00080000:1.0:1788879632.509728:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879632.509732:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879632.509745:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879632.509779:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511648256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879632.509793:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511648128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879632.509799:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879632.509805:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879632.509808:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879632.509813:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4619 00000100:00080000:1.0:1788879632.509816:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879632.509821:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879632.509827:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879632.509842:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511648384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879632.509853:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879632.509858:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879632.509860:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879632.509863:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4619 00000100:00080000:1.0:1788879632.509870:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879632.509874:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879632.509880:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879632.509887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879632.509900:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511648512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879632.509913:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879632.509917:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879632.509919:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879632.509924:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4619 00000100:00080000:1.0:1788879632.509931:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879632.509936:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879632.509943:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879632/real 1788879632] req@ffffa06188699f80 x1875771511648128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879632.509944:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879632.509967:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511648640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879632.509977:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511648128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879632.510013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879632.510020:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879632.510029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511648128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879632.510062:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879632.510093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879632.510108:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879632/real 1788879632] req@ffffa06188698000 x1875771511648256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879632.510123:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511648256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879632.510143:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879632.510147:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879632.510151:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879632.510158:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879632.510177:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879632.510188:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879632/real 1788879632] req@ffffa0618869a680 x1875771511648384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879632.510202:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511648384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879632.510212:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879632.510216:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879632.510219:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879632.510226:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879632.510245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879632.510256:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879632/real 1788879632] req@ffffa0618869bb80 x1875771511648512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879632.510270:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511648512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879632.510279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879632.510282:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879632.510287:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879632.510294:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879632.510311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879632.510324:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879632/real 1788879632] req@ffffa0618869ad80 x1875771511648640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879632.510348:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511648640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879632.510363:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879632.510369:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879632.510376:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879633.149455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879633.149570:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879633.405484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879634.173523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879634.173673:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879634.429511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879635.197478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879635.197661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879635.453524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879636.221473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879636.221635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879636.477520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879637.245516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879637.245631:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879637.501492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879637.629525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879637.629537:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879637.629544:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879637.629561:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4624 00000100:00080000:1.0:1788879637.629578:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879637.629591:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879637.629647:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511648768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879637.629683:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879637.629688:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879637.629691:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879637.629697:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4624 00000100:00080000:1.0:1788879637.629701:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879637.629706:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879637.629744:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879637.629771:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511648768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879637.629774:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511648896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879637.629803:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879637.629808:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879637.629810:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879637.629816:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4624 00000100:00080000:1.0:1788879637.629822:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879637.629827:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879637.629834:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879637.629849:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511649024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879637.629861:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879637.629866:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879637.629869:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879637.629874:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4624 00000100:00080000:1.0:1788879637.629899:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879637.629904:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879637.629910:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879637.629925:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511649152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879637.629938:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879637.629943:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879637.629947:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879637.629955:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4624 00000100:00080000:1.0:1788879637.629961:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879637.629970:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879637.629982:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648896:192.168.203.111@tcp:38:kworker.0 00000008:00100000:1.0:1788879637.629983:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879637.630029:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511649280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879637.630069:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879637.630105:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879637.630135:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879638.269492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879638.269598:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879638.525537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879639.293499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879639.293775:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879639.549504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879640.317489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879640.317608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879640.573470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879640.701825:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879640.701879:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879640.701901:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879640.701950:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879640.701963:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879640.701979:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879640.701991:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879640.702026:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879637/real 1788879640] req@ffffa06188699f80 x1875771511649280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879640.702078:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511649280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879640.702102:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879640.702112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879640.702120:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879640.702136:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879637/real 1788879640] req@ffffa0618869bb80 x1875771511648896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879640.702160:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511648896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879640.702175:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879640.702182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879640.702189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511648896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879640.702201:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879637/real 1788879640] req@ffffa0618869a680 x1875771511649024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879640.702222:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511649024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879640.702237:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879640.702254:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879640.702260:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879640.702279:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879637/real 1788879640] req@ffffa06188698000 x1875771511649152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879640.702306:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511649152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879640.702321:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879640.702327:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879640.702333:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879640.702346:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879637/real 1788879640] req@ffffa0618869ad80 x1875771511648768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879640.702368:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511648768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879640.702420:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879640.702428:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879640.702434:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511648768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879641.341538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879641.341736:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879641.597470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879642.365506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879642.365618:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879642.621492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879642.749568:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879642.749582:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879642.749588:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879642.749602:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4629 00000100:00080000:3.0:1788879642.749617:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879642.749629:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879642.749711:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511649408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879642.749760:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879642.749768:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879642.749772:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879642.749780:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4629 00000100:00080000:3.0:1788879642.749787:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879642.749798:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879642.749817:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879642.749836:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511649408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879642.749852:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511649536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879642.749877:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879642.749884:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879642.749888:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879642.749894:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4629 00000100:00080000:3.0:1788879642.749906:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879642.749915:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879642.749925:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879642.749953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879642.749957:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511649664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879642.750001:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879642.750006:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879642.750009:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879642.750016:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4629 00000100:00080000:3.0:1788879642.750024:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879642.750030:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879642.750040:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879642/real 1788879642] req@ffffa06188871c00 x1875771511649408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879642.750045:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879642.750081:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511649792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879642.750094:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511649408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879642.750150:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879642.750164:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879642.750179:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879642.750195:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4629 00000100:00080000:0.0:1788879642.750196:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879642.750206:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879642.750213:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788879642.750223:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879642.750234:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511649408:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879642.750241:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879642.750274:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511649920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879642.750290:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879642.750348:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879642.750374:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879642/real 1788879642] req@ffffa06188873800 x1875771511649536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879642.750439:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511649536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879642.750457:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879642.750463:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879642.750469:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879642.750482:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879642.750523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879642.750568:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879642/real 1788879642] req@ffffa06188871880 x1875771511649664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879642.750589:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511649664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879642.750602:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879642.750608:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879642.750612:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879642.750624:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879642.750657:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879642.750675:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879642/real 1788879642] req@ffffa06188870380 x1875771511649792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879642.750724:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511649792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879642.750739:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879642.750745:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879642.750751:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879642.750780:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879642.750818:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879642.750840:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879642/real 1788879642] req@ffffa06188870a80 x1875771511649920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879642.750861:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511649920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879642.750876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879642.750882:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879642.750888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511649920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879643.389500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879643.389675:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879643.645505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879644.413479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879644.413617:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879644.669503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879645.437531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879645.437640:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879645.693512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879646.461531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879646.461638:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879646.717595:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879647.485527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879647.485730:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879647.741488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879647.869529:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879647.869538:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879647.869541:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879647.869550:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4634 00000100:00080000:1.0:1788879647.869560:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879647.869566:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879647.869612:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511650048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879647.869640:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879647.869644:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879647.869658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879647.869663:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4634 00000100:00080000:1.0:1788879647.869668:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879647.869673:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879647.869684:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879647.869691:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511650048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879647.869701:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511650176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879647.869710:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879647.869713:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879647.869714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879647.869717:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4634 00000100:00080000:1.0:1788879647.869719:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879647.869722:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879647.869726:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879647.869734:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511650304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879647.869740:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879647.869744:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879647.869745:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879647.869750:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4634 00000100:00080000:1.0:1788879647.869754:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879647.869762:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879647.869763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879647.869772:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879647.869786:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511650432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879647.869798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879647.869800:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879647.869802:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879647.869808:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879647/real 1788879647] req@ffffa06188698380 x1875771511650048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879647.869808:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4634 00000100:00080000:1.0:1788879647.869814:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879647.869819:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879647.869827:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879647.869839:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511650560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879647.869845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511650048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879647.869860:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879647.869865:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879647.869871:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511650048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879647.869895:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879647.869917:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879647.869930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879647/real 1788879647] req@ffffa0618869b800 x1875771511650176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879647.869938:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511650176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879647.869944:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879647.869946:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879647.869949:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879647.869953:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879647.869965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879647.869972:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879647/real 1788879647] req@ffffa06188699c00 x1875771511650304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879647.869984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511650304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879647.869989:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879647.869991:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879647.869993:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879647.869996:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879647.870007:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879647.870013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879647/real 1788879647] req@ffffa0618869aa00 x1875771511650432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879647.870020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511650432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879647.870025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879647.870027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879647.870029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879647.870032:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879647.870041:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879647.870057:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879647/real 1788879647] req@ffffa0618869b100 x1875771511650560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879647.870064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511650560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879647.870073:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879647.870075:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879647.870077:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879648.509483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879648.509569:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879648.765453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879649.533535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879649.533606:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879649.789535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879650.557507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879650.557708:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879650.813554:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879651.581554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879651.581673:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879651.837546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879652.605553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879652.605685:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879652.861515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879652.989738:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879652.989765:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879652.989771:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879652.989798:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4639 00000100:00080000:3.0:1788879652.989815:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879652.989828:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879652.989885:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511650688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879652.989928:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879652.989939:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879652.989942:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879652.989948:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4639 00000100:00080000:3.0:1788879652.989954:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879652.989983:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879652.989999:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879652.990016:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511650688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879652.990025:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511650816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879652.990044:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879652.990049:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879652.990051:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879652.990056:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4639 00000100:00080000:3.0:1788879652.990071:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879652.990077:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879652.990084:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879652.990112:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511650944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879652.990137:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879652.990138:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879652.990148:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879652.990151:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879652.990158:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4639 00000100:00080000:3.0:1788879652.990164:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879652.990171:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879652.990183:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879652.990198:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879652/real 1788879652] req@ffffa06188872d80 x1875771511650688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879652.990226:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511651072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879652.990250:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511650688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879652.990251:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879652.990267:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879652.990276:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879652.990280:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879652.990287:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4639 00000100:00080000:0.0:1788879652.990304:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788879652.990305:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879652.990314:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511650688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879652.990317:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879652.990331:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879652.990349:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511651200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879652.990356:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879652.990421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879652.990438:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879652/real 1788879652] req@ffffa06188873b80 x1875771511650816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879652.990451:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511650816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879652.990501:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879652.990509:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879652.990515:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879652.990528:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879652.990580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879652.990605:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879652/real 1788879652] req@ffffa06188872300 x1875771511650944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879652.990651:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511650944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879652.990668:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879652.990675:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879652.990681:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511650944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879652.990694:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879652.990748:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879652.990772:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879652/real 1788879652] req@ffffa06188872a00 x1875771511651072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879652.990799:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511651072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879652.990817:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879652.990823:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879652.990828:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879652.990854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879652.990882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879652.990902:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879652/real 1788879652] req@ffffa06188870700 x1875771511651200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879652.990926:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511651200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879652.990943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879652.990949:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879652.990956:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879653.629497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879653.629705:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879653.885483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879654.653461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879654.653547:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879654.909486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879655.677538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879655.677651:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879655.933528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879656.701469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879656.701691:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879656.957507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879657.725478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879657.725540:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879657.981482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879658.109531:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879658.109543:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879658.109548:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879658.109560:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4644 00000100:00080000:1.0:1788879658.109573:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879658.109594:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879658.109652:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511651328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879658.109677:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879658.109680:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879658.109681:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879658.109685:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4644 00000100:00080000:1.0:1788879658.109689:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879658.109693:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879658.109703:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879658.109715:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511651456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879658.109716:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511651328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879658.109731:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879658.109733:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879658.109734:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879658.109737:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4644 00000100:00080000:1.0:1788879658.109738:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879658.109741:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879658.109745:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879658.109753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511651584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879658.109759:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879658.109762:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879658.109764:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879658.109766:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4644 00000100:00080000:1.0:1788879658.109777:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879658.109780:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879658.109783:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879658.109788:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879658.109792:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511651712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879658.109800:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879658.109803:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879658.109804:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879658.109807:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4644 00000100:00080000:1.0:1788879658.109809:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879658.109811:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879658.109815:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879658.109826:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879658/real 1788879658] req@ffffa0618869b100 x1875771511651328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879658.109827:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511651840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879658.109865:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511651328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879658.109875:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879658.109878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879658.109881:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511651328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879658.109904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879658.109919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879658.109926:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879658/real 1788879658] req@ffffa0618869aa00 x1875771511651456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879658.109934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511651456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879658.109940:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879658.109942:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879658.109944:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879658.109948:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879658.109958:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879658.109964:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879658/real 1788879658] req@ffffa06188699c00 x1875771511651584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879658.109971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511651584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879658.109976:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879658.109978:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879658.109980:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879658.109984:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879658.109994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879658.110000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879658/real 1788879658] req@ffffa0618869b800 x1875771511651712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879658.110007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511651712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879658.110013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879658.110014:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879658.110016:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879658.110019:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879658.110030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879658.110035:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879658/real 1788879658] req@ffffa06188698380 x1875771511651840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879658.110042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511651840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879658.110047:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879658.110049:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879658.110051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511651840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788879658.749433:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879658.749589:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879659.005461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879659.773489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879659.773652:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879660.029583:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879660.797488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879660.797575:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879661.053507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879661.821563:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879661.821690:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879662.077542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788879662.845539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879662.845685:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879663.101464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879663.229613:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879663.229630:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879663.229638:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879663.229656:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4649 00000100:00080000:2.0:1788879663.229672:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879663.229683:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879663.229735:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511651968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879663.229776:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879663.229781:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879663.229784:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879663.229789:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4649 00000100:00080000:2.0:1788879663.229794:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879663.229800:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879663.229815:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879663.229840:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511652096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879663.229841:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511651968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879663.229869:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879663.229874:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879663.229876:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879663.229880:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4649 00000100:00080000:2.0:1788879663.229885:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879663.229889:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879663.229896:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879663.229915:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511652224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879663.229929:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879663.229934:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879663.229937:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879663.229943:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4649 00000100:00080000:2.0:1788879663.229950:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879663.229956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879663.229966:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879663.229981:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879663.230010:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511652352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879663.230013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879663/real 1788879663] req@ffffa061888ace00 x1875771511651968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879663.230078:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879663.230086:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879663.230090:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879663.230099:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4649 00000100:00100000:0.0:1788879663.230104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511651968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879663.230108:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879663.230126:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879663.230132:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879663.230141:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879663.230147:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511651968:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788879663.230151:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879663.230175:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511652480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879663.230187:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879663.230227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879663.230246:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879663/real 1788879663] req@ffffa061888ad180 x1875771511652096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879663.230261:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511652096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879663.230272:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879663.230276:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879663.230280:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879663.230287:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879663.230307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879663.230318:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879663/real 1788879663] req@ffffa061888ae680 x1875771511652224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879663.230332:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511652224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879663.230350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879663.230354:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879663.230358:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879663.230365:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879663.230416:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879663.230432:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879663/real 1788879663] req@ffffa061888ad880 x1875771511652352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879663.230446:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511652352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879663.230456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879663.230460:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879663.230464:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879663.230485:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879663.230507:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879663.230519:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879663/real 1788879663] req@ffffa061888ac380 x1875771511652480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879663.230533:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511652480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879663.230570:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879663.230574:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879663.230579:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788879663.869605:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879663.869611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879664.125563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788879664.893500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879664.893627:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879665.149601:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879665.917553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879665.917592:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879666.173488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879666.941527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879666.941645:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879667.197581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879667.965590:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879667.965652:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879668.221564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879668.349492:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879668.349517:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879668.349522:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879668.349533:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4655 00000100:00080000:2.0:1788879668.349545:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879668.349553:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879668.349595:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511652608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879668.349634:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879668.349639:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879668.349642:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879668.349653:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4655 00000100:00080000:2.0:1788879668.349661:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879668.349669:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879668.349682:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879668.349705:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511652736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879668.349723:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879668.349726:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879668.349728:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879668.349732:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4655 00000100:00080000:2.0:1788879668.349739:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788879668.349764:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511652608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879668.349766:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879668.349783:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879668.349800:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511652864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879668.349811:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879668.349815:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879668.349817:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879668.349821:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4655 00000100:00080000:2.0:1788879668.349824:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879668.349828:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879668.349833:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879668.349846:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511652992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879668.349854:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879668.349857:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879668.349860:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879668.349863:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4655 00000100:00080000:2.0:1788879668.349867:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879668.349870:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879668.349876:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1788879668.349882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879668.349889:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511653120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879668.349955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879668/real 1788879668] req@ffffa061888ac700 x1875771511652608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879668.349981:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511652608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879668.349996:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879668.350002:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879668.350008:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511652608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879668.350043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879668.350071:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879668.350086:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879668/real 1788879668] req@ffffa061888aed80 x1875771511652736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879668.350111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511652736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879668.350130:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879668.350134:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879668.350139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879668.350146:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879668.350165:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879668.350176:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879668/real 1788879668] req@ffffa061888adc00 x1875771511652864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879668.350191:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511652864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879668.350200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879668.350204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879668.350207:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879668.350214:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879668.350234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879668.350245:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879668/real 1788879668] req@ffffa061888afb80 x1875771511652992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879668.350259:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511652992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879668.350278:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879668.350281:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879668.350285:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511652992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879668.350291:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879668.350310:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879668.350321:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879668/real 1788879668] req@ffffa061888aea00 x1875771511653120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879668.350333:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511653120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879668.350357:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879668.350361:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879668.350365:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879668.989576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879668.989698:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879669.245626:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879670.013563:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879670.013741:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879670.269593:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879671.037520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879671.037681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879671.293633:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879672.061461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879672.061605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879672.317569:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879673.085532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879673.085643:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879673.341552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879673.469570:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879673.469584:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879673.469591:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879673.469606:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4660 00000100:00080000:3.0:1788879673.469622:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879673.469633:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879673.469684:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511653248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879673.469723:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879673.469728:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879673.469732:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879673.469738:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4660 00000100:00080000:3.0:1788879673.469742:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879673.469747:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879673.469761:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879673.469778:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511653376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879673.469794:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511653248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879673.469796:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879673.469810:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879673.469816:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879673.469827:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4660 00000100:00080000:3.0:1788879673.469830:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879673.469835:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879673.469841:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879673.469857:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511653504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879673.469869:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879673.469874:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879673.469877:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879673.469895:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4660 00000100:00080000:3.0:1788879673.469900:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879673.469909:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:1.0:1788879673.469914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788879673.469918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879673.469935:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511653632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879673.469963:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879673.469967:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879673.469970:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879673.469975:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4660 00000100:00080000:3.0:1788879673.469979:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879673.469984:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879673.469990:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879673.470010:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511653760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879673.470014:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879673/real 1788879673] req@ffffa06188873480 x1875771511653248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879673.470055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511653248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879673.470079:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879673.470084:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879673.470089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511653248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879673.470117:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879673.470147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879673.470162:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879673/real 1788879673] req@ffffa06188872680 x1875771511653376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879673.470176:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511653376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879673.470186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879673.470190:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879673.470193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879673.470200:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879673.470217:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879673.470226:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879673/real 1788879673] req@ffffa06188873100 x1875771511653504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879673.470246:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511653504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879673.470255:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879673.470257:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879673.470261:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879673.470266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879673.470283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879673.470293:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879673/real 1788879673] req@ffffa06188871500 x1875771511653632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879673.470305:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511653632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879673.470314:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879673.470316:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879673.470320:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879673.470325:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879673.470340:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879673.470349:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879673/real 1788879673] req@ffffa06188870700 x1875771511653760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879673.470361:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511653760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879673.470377:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879673.470424:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879673.470446:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511653760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879674.109539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879674.109668:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879674.365536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879675.133518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879675.133711:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879675.389549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879676.157495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879676.157523:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879676.413493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879677.181515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879677.181586:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879677.437535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879678.205547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879678.205557:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879678.461528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879678.589503:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879678.589512:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879678.589517:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879678.589527:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4665 00000100:00080000:3.0:1788879678.589537:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879678.589544:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879678.589580:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511653888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879678.589605:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879678.589608:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879678.589609:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879678.589612:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4665 00000100:00080000:3.0:1788879678.589614:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879678.589617:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879678.589636:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879678.589645:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511654016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879678.589652:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879678.589655:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879678.589656:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879678.589658:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4665 00000100:00080000:3.0:1788879678.589660:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879678.589662:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879678.589666:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879678.589668:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511653888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879678.589675:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511654144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879678.589682:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879678.589696:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879678.589698:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879678.589700:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4665 00000100:00080000:3.0:1788879678.589702:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879678.589705:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879678.589708:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879678.589718:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511654272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879678.589724:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879678.589727:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879678.589728:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879678.589730:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4665 00000100:00080000:3.0:1788879678.589733:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879678.589736:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879678.589739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879678.589748:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511654400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879678.589749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879678.589785:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879678/real 1788879678] req@ffffa06188872a00 x1875771511653888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879678.589811:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511653888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879678.589820:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879678.589823:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879678.589826:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511653888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879678.589831:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879678.589844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879678.589852:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879678/real 1788879678] req@ffffa06188872300 x1875771511654016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879678.589860:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511654016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879678.589865:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879678.589867:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879678.589870:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879678.589889:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879678.589899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879678.589905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879678/real 1788879678] req@ffffa06188873b80 x1875771511654144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879678.589912:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511654144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879678.589917:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879678.589919:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879678.589921:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879678.589925:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879678.589934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879678.589940:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879678/real 1788879678] req@ffffa06188872d80 x1875771511654272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879678.589947:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511654272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879678.589953:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879678.589955:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879678.589957:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879678.589960:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879678.589970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879678.589975:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879678/real 1788879678] req@ffffa06188870a80 x1875771511654400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879678.589982:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511654400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879678.589988:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879678.589989:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879678.589991:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879679.229545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879679.229734:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879679.485563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879680.253578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879680.253703:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879680.509492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879681.277467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879681.277551:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879681.533483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879682.301512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879682.301625:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879682.557533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879683.325496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879683.325633:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879683.581566:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879683.709523:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879683.709537:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879683.709542:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879683.709555:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4670 00000100:00080000:3.0:1788879683.709569:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879683.709578:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879683.709625:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511654528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879683.709659:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879683.709664:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879683.709666:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879683.709670:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4670 00000100:00080000:3.0:1788879683.709674:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879683.709678:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879683.709691:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879683.709705:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511654656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879683.709720:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511654528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879683.709723:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879683.709736:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879683.709742:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879683.709747:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4670 00000100:00080000:3.0:1788879683.709750:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879683.709754:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879683.709760:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879683.709773:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511654784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879683.709784:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879683.709788:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879683.709790:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879683.709795:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4670 00000100:00080000:3.0:1788879683.709799:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879683.709803:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879683.709809:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879683.709826:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511654912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879683.709827:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879683.709843:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879683.709848:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879683.709851:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879683.709857:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4670 00000100:00080000:3.0:1788879683.709862:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879683.709868:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879683.709885:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879683.709894:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879683/real 1788879683] req@ffffa06188870380 x1875771511654528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879683.709902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511655040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879683.709938:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511654528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879683.709977:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879683.709983:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879683.709988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511654528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879683.710017:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879683.710043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879683.710057:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879683/real 1788879683] req@ffffa06188871880 x1875771511654656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879683.710071:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511654656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879683.710080:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879683.710084:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879683.710088:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879683.710094:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788879683.710111:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879683.710121:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879683/real 1788879683] req@ffffa06188873800 x1875771511654784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879683.710151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511654784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879683.710159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879683.710162:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879683.710165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879683.710171:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879683.710187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879683.710206:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879683/real 1788879683] req@ffffa06188871c00 x1875771511654912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879683.710218:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511654912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879683.710227:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879683.710229:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879683.710233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511654912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879683.710238:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788879683.710256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879683.710269:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879683/real 1788879683] req@ffffa06188870a80 x1875771511655040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879683.710286:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511655040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879683.710302:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879683.710305:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788879683.710308:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879684.349527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879684.349661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879684.605583:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879685.373495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879685.373602:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879685.629553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879686.397544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879686.397638:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879686.653540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879687.421544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879687.421613:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879687.677508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879688.445525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879688.445647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879688.701509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879688.829592:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879688.829605:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879688.829611:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879688.829625:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4675 00000100:00080000:1.0:1788879688.829640:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879688.829651:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879688.829700:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511655168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879688.829742:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879688.829747:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879688.829749:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879688.829754:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4675 00000100:00080000:1.0:1788879688.829760:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879688.829765:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879688.829778:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879688.829796:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511655296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879688.829812:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879688.829816:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511655168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879688.829818:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879688.829826:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879688.829839:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4675 00000100:00080000:1.0:1788879688.829844:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879688.829848:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879688.829854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879688.829870:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511655424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879688.829883:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879688.829888:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879688.829890:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879688.829896:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4675 00000100:00080000:1.0:1788879688.829919:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879688.829924:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879688.829931:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879688.829935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879688.829953:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511655552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879688.829977:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879688.829982:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879688.829985:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879688.829994:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4675 00000100:00080000:1.0:1788879688.829998:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879688.830002:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879688.830007:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879688.830017:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879688/real 1788879688] req@ffffa0618869ad80 x1875771511655168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879688.830020:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511655680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879688.830100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511655168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879688.830116:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879688.830121:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879688.830127:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511655168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879688.830159:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879688.830195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879688.830212:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879688/real 1788879688] req@ffffa06188698000 x1875771511655296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879688.830244:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511655296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879688.830254:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879688.830259:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879688.830262:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879688.830269:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879688.830289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879688.830300:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879688/real 1788879688] req@ffffa0618869a680 x1875771511655424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879688.830313:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511655424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879688.830329:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879688.830333:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879688.830336:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879688.830343:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879688.830415:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879688.830433:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879688/real 1788879688] req@ffffa0618869bb80 x1875771511655552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879688.830446:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511655552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879688.830456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879688.830459:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879688.830463:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879688.830469:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879688.830502:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879688.830523:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879688/real 1788879688] req@ffffa06188699f80 x1875771511655680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879688.830544:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511655680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879688.830560:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879688.830566:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879688.830573:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879689.469449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879689.469512:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879689.725452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879690.493537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879690.493665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879690.749513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879691.517536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879691.517652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879691.773520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879692.541509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879692.541594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879692.797481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879693.565500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879693.565729:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879693.821542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879693.949565:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879693.949582:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879693.949588:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879693.949604:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4680 00000100:00080000:3.0:1788879693.949620:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879693.949632:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879693.949687:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511655808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879693.949735:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879693.949741:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879693.949744:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879693.949751:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4680 00000100:00080000:3.0:1788879693.949757:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879693.949764:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511655808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879693.949765:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879693.949785:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879693.949807:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511655936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879693.949823:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879693.949831:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879693.949833:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879693.949839:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4680 00000100:00080000:3.0:1788879693.949844:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879693.949845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879693.949855:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879693.949864:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879693.949883:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511656064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879693.949892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879693/real 1788879693] req@ffffa06188872d80 x1875771511655808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879693.949903:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879693.949911:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879693.949914:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879693.949917:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511655808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879693.949924:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4680 00000100:00080000:0.0:1788879693.949935:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879693.949936:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879693.949946:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879693.949954:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879693.949955:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879693.949962:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511655808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879693.949985:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511656192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879693.949992:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655936:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788879693.950012:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879693.950018:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879693.950022:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879693.950025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879693.950032:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4680 00000100:00080000:3.0:1788879693.950039:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879693.950041:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879693/real 1788879693] req@ffffa06188873b80 x1875771511655936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879693.950051:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879693.950059:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511655936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879693.950061:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879693.950076:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788879693.950087:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511656320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879693.950088:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879693.950098:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511655936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879693.950112:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879693.950132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879693.950142:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879693/real 1788879693] req@ffffa06188872300 x1875771511656064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879693.950153:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511656064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879693.950160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879693.950162:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879693.950165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879693.950177:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879693.950191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879693.950207:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879693/real 1788879693] req@ffffa06188872a00 x1875771511656192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879693.950225:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511656192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879693.950232:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879693.950235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879693.950238:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879693.950242:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879693.950254:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879693.950263:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879693/real 1788879693] req@ffffa06188870700 x1875771511656320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879693.950272:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511656320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879693.950279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879693.950281:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879693.950283:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879694.589532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879694.589689:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879694.845522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879695.613561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879695.613737:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879695.869500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879696.637527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879696.637614:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879696.893497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879697.661548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879697.661665:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879697.917502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879698.685485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879698.685529:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879698.941498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879699.069510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879699.069522:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879699.069557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879699.069576:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4685 00000100:00080000:2.0:1788879699.069594:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879699.069609:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879699.069681:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511656448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879699.069703:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879699.069706:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879699.069707:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879699.069710:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4685 00000100:00080000:2.0:1788879699.069714:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879699.069736:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879699.069751:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879699.069782:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511656448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879699.069788:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511656576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879699.069812:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879699.069816:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879699.069819:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879699.069826:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4685 00000100:00080000:2.0:1788879699.069832:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879699.069836:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879699.069843:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879699.069867:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511656704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879699.069902:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879699.069906:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879699.069908:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879699.069912:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4685 00000100:00080000:2.0:1788879699.069917:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879699.069922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879699.069926:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879699.069933:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879699.069946:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511656832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879699.069980:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879699.069981:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879699/real 1788879699] req@ffffa061888ae300 x1875771511656448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879699.069999:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879699.070003:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879699.070010:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4685 00000100:00100000:0.0:1788879699.070011:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511656448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879699.070016:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879699.070027:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879699.070054:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788879699.070056:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879699.070069:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879699.070077:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511656448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879699.070085:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511656960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879699.070131:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879699.070149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879699.070158:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879699/real 1788879699] req@ffffa061888af100 x1875771511656576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879699.070166:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511656576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879699.070172:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879699.070174:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879699.070177:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879699.070180:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879699.070191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879699.070196:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879699/real 1788879699] req@ffffa061888ad500 x1875771511656704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879699.070204:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511656704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879699.070209:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879699.070210:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879699.070212:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879699.070216:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879699.070226:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879699.070232:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879699/real 1788879699] req@ffffa061888ac000 x1875771511656832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879699.070239:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511656832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879699.070244:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879699.070246:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879699.070248:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879699.070251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879699.070261:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879699.070266:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879699/real 1788879699] req@ffffa061888adf80 x1875771511656960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879699.070273:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511656960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879699.070278:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879699.070280:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879699.070282:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511656960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879699.709541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879699.709669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879699.965503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879700.733519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879700.733642:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879700.989509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879701.757457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879701.757578:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879702.013545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879702.781454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879702.781608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879703.037469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879703.805465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879703.805539:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879704.061474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879704.189479:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879704.189488:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879704.189492:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879704.189501:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4690 00000100:00080000:1.0:1788879704.189512:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879704.189519:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879704.189556:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511657088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879704.189579:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879704.189581:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879704.189583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879704.189586:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4690 00000100:00080000:1.0:1788879704.189588:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879704.189591:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879704.189600:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879704.189613:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511657216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879704.189614:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511657088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879704.189629:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879704.189632:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879704.189633:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879704.189635:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4690 00000100:00080000:1.0:1788879704.189637:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879704.189640:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879704.189643:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879704.189651:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511657344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879704.189657:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879704.189659:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879704.189660:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879704.189663:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4690 00000100:00080000:1.0:1788879704.189676:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879704.189679:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879704.189683:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879704.189691:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511657472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879704.189697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879704.189700:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879704.189702:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879704.189706:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4690 00000100:00080000:1.0:1788879704.189709:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879704.189712:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879704.189717:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879704.189720:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879704.189728:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511657600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879704.189752:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879704.189769:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879704.189785:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879704.829495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879704.829636:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879705.085557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879705.853533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879705.853708:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879706.109595:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879706.877506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879706.877615:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879707.133497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879707.261724:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879707.261769:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879707.261786:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879707.261826:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879707.261835:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879707.261842:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879707.261849:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879707.261907:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879704/real 1788879707] req@ffffa0618869ad80 x1875771511657600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879707.261939:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511657600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879707.261959:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879707.261974:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879707.261979:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879707.261988:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879704/real 1788879707] req@ffffa0618869bb80 x1875771511657216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879707.262004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511657216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879707.262018:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879707.262023:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879707.262028:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879707.262038:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879704/real 1788879707] req@ffffa0618869a680 x1875771511657344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879707.262055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511657344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879707.262066:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879707.262070:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879707.262075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879707.262085:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879704/real 1788879707] req@ffffa06188698000 x1875771511657472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879707.262108:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511657472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879707.262118:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879707.262123:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879707.262127:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879707.262136:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879704/real 1788879707] req@ffffa06188699f80 x1875771511657088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879707.262152:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511657088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879707.262161:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879707.262166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879707.262171:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511657088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879707.901494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879707.901550:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879708.157524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879708.925451:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879708.925515:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879709.181525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879709.309694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879709.309708:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879709.309713:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879709.309728:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4696 00000100:00080000:1.0:1788879709.309743:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879709.309754:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879709.309820:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511657728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879709.309911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511657728:192.168.203.111@tcp:250:kworker.0 00000800:00000100:0.0:1788879709.309994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879709.310039:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879709.310045:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879709.310049:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879709.310056:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4696 00000100:00080000:1.0:1788879709.310061:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879709.310068:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879709.310083:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879709.310102:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511657856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879709.310182:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879709/real 1788879709] req@ffffa06188698380 x1875771511657728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879709.310210:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511657728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879709.310239:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879709.310245:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879709.310250:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511657728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879709.310285:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879709.310320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879709.310338:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879709/real 1788879709] req@ffffa0618869b800 x1875771511657856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879709.310356:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511657856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879709.310368:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879709.310373:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879709.310378:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657856:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879709.310494:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879709.310501:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879709.310503:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879709.310509:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4696 00000100:00080000:1.0:1788879709.310516:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879709.310522:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879709.310531:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879709.310550:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511657984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879709.310563:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879709.310567:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879709.310570:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879709.310575:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4696 00000100:00080000:1.0:1788879709.310579:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879709.310583:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879709.310590:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879709.310605:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511658112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879709.310616:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879709.310621:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879709.310623:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879709.310627:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4696 00000100:00080000:1.0:1788879709.310632:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879709.310640:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879709.310645:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879709.310657:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511658240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879709.310748:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879709.310789:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879709.310808:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879709/real 1788879709] req@ffffa06188699c00 x1875771511657984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879709.310826:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511657984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879709.310847:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879709.310852:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879709.310858:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511657984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879709.310867:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879709.310902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879709.310919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879709/real 1788879709] req@ffffa0618869aa00 x1875771511658112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879709.310936:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511658112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879709.310953:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879709.310958:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879709.310962:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879709.310970:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879709.310994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879709.311008:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879709/real 1788879709] req@ffffa0618869b100 x1875771511658240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879709.311021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511658240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879709.311030:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879709.311034:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879709.311038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879709.949464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879709.949568:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879710.205589:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879710.973519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879710.973580:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879711.229489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879711.997467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879711.997586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879712.253514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879713.021513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879713.021634:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879713.277459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879714.045465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879714.045540:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879714.301455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879714.429520:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879714.429529:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879714.429532:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879714.429541:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4701 00000100:00080000:2.0:1788879714.429551:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879714.429557:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879714.429593:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511658368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879714.429615:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879714.429618:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879714.429620:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879714.429623:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4701 00000100:00080000:2.0:1788879714.429626:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879714.429629:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879714.429638:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879714.429648:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511658368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879714.429649:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511658496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879714.429661:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879714.429664:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879714.429665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879714.429667:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4701 00000100:00080000:2.0:1788879714.429669:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879714.429672:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879714.429675:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879714.429685:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511658624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879714.429691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879714.429694:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879714.429695:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879714.429698:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4701 00000100:00080000:2.0:1788879714.429701:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879714.429704:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879714.429707:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879714.429715:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511658752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879714.429723:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879714.429724:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879714.429728:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879714.429730:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879714.429733:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4701 00000100:00080000:2.0:1788879714.429736:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879714.429738:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879714.429741:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879714.429748:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511658880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879714.429764:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879714/real 1788879714] req@ffffa061888af480 x1875771511658368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879714.429780:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511658368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879714.429788:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879714.429791:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879714.429794:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511658368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879714.429814:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879714.429830:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879714.429838:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879714/real 1788879714] req@ffffa061888aca80 x1875771511658496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879714.429845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511658496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879714.429851:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879714.429853:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879714.429856:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879714.429860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879714.429871:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879714.429877:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879714/real 1788879714] req@ffffa061888adf80 x1875771511658624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879714.429884:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511658624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879714.429889:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879714.429891:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879714.429892:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879714.429896:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879714.429906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879714.429912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879714/real 1788879714] req@ffffa061888ac000 x1875771511658752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879714.429918:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511658752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879714.429924:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879714.429925:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879714.429927:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879714.429930:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879714.429939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879714.429945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879714/real 1788879714] req@ffffa061888ad500 x1875771511658880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879714.429953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511658880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879714.429958:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879714.429959:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879714.429961:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511658880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879715.069464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879715.069578:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879715.325503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879716.093496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879716.093611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879716.349557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879717.117454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879717.117533:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879717.373438:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879718.141500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879718.141552:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879718.397477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879719.165454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879719.165537:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879719.421480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879719.549497:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879719.549510:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879719.549517:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879719.549544:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4706 00000100:00080000:3.0:1788879719.549562:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879719.549578:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879719.549661:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511659008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879719.549686:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879719.549689:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879719.549691:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879719.549694:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4706 00000100:00080000:3.0:1788879719.549698:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879719.549702:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879719.549712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879719.549718:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511659008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879719.549725:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511659136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879719.549734:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879719.549737:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879719.549738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879719.549741:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4706 00000100:00080000:3.0:1788879719.549744:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879719.549746:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879719.549750:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879719.549759:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511659264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879719.549765:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879719.549767:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879719.549769:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879719.549771:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4706 00000100:00080000:3.0:1788879719.549774:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879719.549776:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879719.549779:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879719.549783:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879719.549790:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511659392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879719.549797:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879719.549801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879719.549802:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879719.549804:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4706 00000100:00080000:3.0:1788879719.549807:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879719.549809:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879719.549813:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879719.549819:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879719/real 1788879719] req@ffffa06188871500 x1875771511659008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879719.549824:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511659520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879719.549846:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511659008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879719.549857:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879719.549860:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879719.549864:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511659008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879719.549882:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879719.549897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879719.549905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879719/real 1788879719] req@ffffa06188873100 x1875771511659136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879719.549913:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511659136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879719.549919:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879719.549921:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879719.549923:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879719.549927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879719.549937:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879719.549942:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879719/real 1788879719] req@ffffa06188872680 x1875771511659264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879719.549950:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511659264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879719.549955:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879719.549957:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879719.549958:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879719.549962:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879719.549972:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879719.549978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879719/real 1788879719] req@ffffa06188873480 x1875771511659392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879719.549985:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511659392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879719.549991:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879719.549993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879719.549995:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879719.549998:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879719.550007:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879719.550012:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879719/real 1788879719] req@ffffa06188870700 x1875771511659520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879719.550019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511659520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879719.550024:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879719.550027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879719.550047:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879720.189432:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879720.189538:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879720.445563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879721.213504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879721.213661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879721.469539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879722.237509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879722.237625:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879722.493493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879723.261499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879723.261564:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879723.517520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879724.285474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879724.285530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879724.541458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879724.669519:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879724.669531:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879724.669537:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879724.669549:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4711 00000100:00080000:1.0:1788879724.669561:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879724.669569:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879724.669618:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511659648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879724.669642:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879724.669646:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879724.669649:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879724.669654:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4711 00000100:00080000:1.0:1788879724.669659:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879724.669663:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879724.669675:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879724.669678:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511659648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879724.669688:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511659776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879724.669699:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879724.669702:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879724.669703:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879724.669707:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4711 00000100:00080000:1.0:1788879724.669710:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879724.669714:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879724.669718:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879724.669727:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511659904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879724.669735:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879724.669738:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879724.669740:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879724.669744:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4711 00000100:00080000:1.0:1788879724.669747:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879724.669749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879724.669751:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879724.669756:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879724.669764:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511660032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879724.669773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879724.669775:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879724.669777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879724.669781:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4711 00000100:00080000:1.0:1788879724.669784:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879724.669788:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879724/real 1788879724] req@ffffa0618869b100 x1875771511659648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879724.669788:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879724.669792:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879724.669802:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511660160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879724.669807:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511659648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879724.669819:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879724.669823:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879724.669828:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511659648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879724.669854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879724.669874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879724.669885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879724/real 1788879724] req@ffffa0618869aa00 x1875771511659776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879724.669893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511659776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879724.669899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879724.669901:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879724.669904:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879724.669908:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879724.669919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879724.669925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879724/real 1788879724] req@ffffa06188699c00 x1875771511659904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879724.669932:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511659904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879724.669938:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879724.669939:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879724.669941:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511659904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879724.669945:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879724.669955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879724.669961:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879724/real 1788879724] req@ffffa0618869b800 x1875771511660032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879724.669968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511660032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879724.669973:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879724.669975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879724.669977:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879724.669980:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879724.669989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879724.669994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879724/real 1788879724] req@ffffa06188698380 x1875771511660160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879724.670002:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511660160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879724.670007:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879724.670009:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879724.670010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879725.309464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879725.309545:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879725.565502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879726.333466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879726.333556:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879726.589444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879727.357482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879727.357564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879727.613471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879728.381517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879728.381632:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879728.637452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879729.405442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879729.405558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879729.661498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879729.789485:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879729.789494:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879729.789498:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879729.789507:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4716 00000100:00080000:3.0:1788879729.789517:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879729.789525:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879729.789559:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511660288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879729.789585:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879729.789590:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879729.789592:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879729.789598:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4716 00000100:00080000:3.0:1788879729.789602:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879729.789608:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879729.789616:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511660288:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879729.789619:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879729.789634:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511660416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879729.789643:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879729.789646:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879729.789647:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879729.789650:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4716 00000100:00080000:3.0:1788879729.789653:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879729.789656:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879729.789661:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879729.789668:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511660544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879729.789675:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879729.789677:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879729.789681:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879729.789682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879729.789686:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4716 00000100:00080000:3.0:1788879729.789689:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879729.789692:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879729.789696:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879729.789704:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511660672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879729.789711:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879729.789713:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879729.789714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879729.789718:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879729/real 1788879729] req@ffffa06188872a00 x1875771511660288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879729.789721:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4716 00000100:00080000:3.0:1788879729.789726:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879729.789732:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879729.789738:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879729.789746:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511660288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879729.789747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511660800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879729.789774:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879729.789779:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879729.789783:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511660288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879729.789803:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879729.789818:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879729.789826:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879729/real 1788879729] req@ffffa06188872300 x1875771511660416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879729.789834:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511660416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879729.789840:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879729.789842:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879729.789844:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879729.789848:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879729.789857:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879729.789863:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879729/real 1788879729] req@ffffa06188873b80 x1875771511660544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879729.789871:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511660544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879729.789876:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879729.789878:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879729.789880:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879729.789883:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879729.789895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879729.789901:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879729/real 1788879729] req@ffffa06188872d80 x1875771511660672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879729.789908:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511660672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879729.789913:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879729.789915:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879729.789917:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879729.789920:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879729.789930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879729.789935:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879729/real 1788879729] req@ffffa06188870a80 x1875771511660800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879729.789942:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511660800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879729.789952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879729.789954:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879729.789956:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511660800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879730.429485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879730.429604:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879730.685446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879731.453551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879731.453665:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879731.709496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879732.477503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879732.477683:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879732.733530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879733.501454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879733.501500:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879733.757533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879734.525490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879734.525591:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879734.781521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879734.909543:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879734.909557:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879734.909565:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879734.909582:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4721 00000100:00080000:3.0:1788879734.909598:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879734.909610:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879734.909664:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511660928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879734.909717:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879734.909720:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879734.909721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879734.909724:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4721 00000100:00080000:3.0:1788879734.909727:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879734.909731:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879734.909740:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879734.909751:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511661056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879734.909762:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511660928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879734.909762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879734.909767:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879734.909770:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879734.909794:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4721 00000100:00080000:3.0:1788879734.909799:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879734.909803:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879734.909810:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879734.909829:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511661184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879734.909851:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879734.909856:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879734.909858:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879734.909862:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4721 00000100:00080000:3.0:1788879734.909868:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879734.909872:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879734.909879:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879734.909888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879734.909899:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511661312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879734.909914:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879734.909919:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879734.909921:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879734.909929:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4721 00000100:00080000:3.0:1788879734.909934:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879734.909939:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879734.909946:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879734.909947:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879734/real 1788879734] req@ffffa06188871c00 x1875771511660928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879734.909969:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511661440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879734.909984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511660928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879734.910003:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879734.910009:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879734.910016:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511660928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879734.910091:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879734.910111:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879734.910120:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879734/real 1788879734] req@ffffa06188873800 x1875771511661056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879734.910133:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511661056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879734.910140:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879734.910142:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879734.910144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879734.910148:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879734.910160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879734.910166:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879734/real 1788879734] req@ffffa06188871880 x1875771511661184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879734.910173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511661184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879734.910179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879734.910180:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879734.910182:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879734.910186:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879734.910196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879734.910202:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879734/real 1788879734] req@ffffa06188870380 x1875771511661312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879734.910210:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511661312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879734.910219:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879734.910222:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879734.910224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879734.910227:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879734.910236:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879734.910242:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879734/real 1788879734] req@ffffa06188870a80 x1875771511661440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879734.910251:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511661440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879734.910260:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879734.910264:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879734.910267:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879735.549475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879735.549544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879735.805501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879736.573470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879736.573514:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879736.829447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879737.597510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879737.597735:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879737.853504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879738.621492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879738.621573:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879738.877452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879739.645524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879739.645603:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879739.901482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879740.029534:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879740.029574:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879740.029582:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879740.029598:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4726 00000100:00080000:1.0:1788879740.029615:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879740.029628:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879740.029686:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511661568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879740.029732:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879740.029739:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879740.029743:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879740.029752:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4726 00000100:00080000:1.0:1788879740.029759:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879740.029768:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879740.029783:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879740.029787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511661568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879740.029805:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511661696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879740.029824:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879740.029829:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879740.029833:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879740.029854:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4726 00000100:00080000:1.0:1788879740.029871:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879740.029878:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879740.029882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879740.029888:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879740.029907:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511661824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879740.029931:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879740/real 1788879740] req@ffffa06188699f80 x1875771511661568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879740.029935:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879740.029942:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879740.029946:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879740.029953:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4726 00000100:00080000:1.0:1788879740.029958:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879740.029964:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879740.029972:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879740.029985:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511661568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879740.030018:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511661952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879740.030025:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879740.030033:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788879740.030034:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879740.030051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511661568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879740.030052:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879740.030056:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879740.030061:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4726 00000100:00080000:1.0:1788879740.030067:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879740.030074:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879740.030081:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879740.030088:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879740.030099:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511662080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879740.030127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879740.030151:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879740/real 1788879740] req@ffffa06188698000 x1875771511661696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879740.030176:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511661696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879740.030192:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879740.030196:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879740.030201:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879740.030220:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879740.030250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879740.030274:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879740/real 1788879740] req@ffffa0618869a680 x1875771511661824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879740.030288:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511661824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879740.030305:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879740.030309:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879740.030313:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879740.030320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879740.030339:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879740.030349:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879740/real 1788879740] req@ffffa0618869bb80 x1875771511661952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879740.030362:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511661952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879740.030372:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879740.030375:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879740.030379:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511661952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879740.030437:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879740.030470:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879740.030483:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879740/real 1788879740] req@ffffa0618869ad80 x1875771511662080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879740.030496:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511662080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879740.030506:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879740.030510:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879740.030515:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879740.669497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879740.669678:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879740.925526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879741.693451:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879741.693524:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879741.949531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879742.717536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879742.717631:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879742.973510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879743.741459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879743.741530:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879743.997450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879744.765496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879744.765647:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879745.021489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879745.149546:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879745.149561:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879745.149568:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879745.149584:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4731 00000100:00080000:3.0:1788879745.149603:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879745.149613:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879745.149670:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511662208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879745.149709:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879745.149714:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879745.149717:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879745.149723:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4731 00000100:00080000:3.0:1788879745.149728:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879745.149734:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879745.149749:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879745.149767:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511662208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879745.149771:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511662336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879745.149794:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879745.149798:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879745.149800:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879745.149806:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4731 00000100:00080000:3.0:1788879745.149810:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879745.149814:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879745.149820:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879745.149836:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511662464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879745.149848:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879745.149854:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879745.149857:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879745.149862:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4731 00000100:00080000:3.0:1788879745.149868:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879745.149875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879745.149880:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879745.149889:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879745.149905:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511662592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879745.149919:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879745.149923:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879745.149925:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879745.149931:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4731 00000100:00100000:0.0:1788879745.149938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879745/real 1788879745] req@ffffa06188872d80 x1875771511662208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879745.149940:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879745.149955:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879745.149988:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879745.150012:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511662720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879745.150019:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511662208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879745.150051:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879745.150061:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879745.150070:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511662208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879745.150109:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879745.150145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879745.150162:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879745/real 1788879745] req@ffffa06188873b80 x1875771511662336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879745.150178:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511662336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879745.150190:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879745.150195:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879745.150200:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879745.150208:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879745.150230:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879745.150242:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879745/real 1788879745] req@ffffa06188872300 x1875771511662464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879745.150256:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511662464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879745.150266:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879745.150270:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879745.150273:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879745.150281:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879745.150302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879745.150314:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879745/real 1788879745] req@ffffa06188872a00 x1875771511662592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879745.150328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511662592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879745.150338:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879745.150341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879745.150345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879745.150351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879745.150369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879745.150412:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879745/real 1788879745] req@ffffa06188870700 x1875771511662720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879745.150429:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511662720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879745.150440:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879745.150444:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879745.150447:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879745.789519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879745.789599:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879746.045497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879746.813497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879746.813633:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879747.069546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879747.837558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879747.837661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879748.093535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879748.861474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879748.861557:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879749.117521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879749.885530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879749.885713:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879750.141529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879750.269532:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879750.269545:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879750.269551:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879750.269566:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4736 00000100:00080000:3.0:1788879750.269582:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879750.269594:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879750.269656:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511662848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879750.269705:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879750.269710:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879750.269713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879750.269718:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4736 00000100:00080000:3.0:1788879750.269723:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879750.269729:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879750.269743:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879750.269760:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511662976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879750.269768:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511662848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879750.269793:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879750.269798:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879750.269800:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879750.269806:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4736 00000100:00080000:3.0:1788879750.269811:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879750.269817:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879750.269823:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879750.269849:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511663104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879750.269861:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879750.269866:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879750.269868:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879750.269874:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4736 00000100:00080000:3.0:1788879750.269880:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879750.269884:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879750.269890:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879750.269909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879750.269910:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511663232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879750.269959:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879750.269964:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879750.269967:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879750.269974:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4736 00000100:00080000:3.0:1788879750.269979:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879750.269985:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879750.269991:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879750.270003:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879750/real 1788879750] req@ffffa06188873480 x1875771511662848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879750.270013:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511663360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879750.270078:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511662848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879750.270097:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879750.270103:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879750.270109:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511662848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879750.270138:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879750.270175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879750.270192:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879750/real 1788879750] req@ffffa06188872680 x1875771511662976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879750.270208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511662976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879750.270218:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879750.270222:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879750.270227:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511662976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879750.270234:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879750.270253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879750.270264:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879750/real 1788879750] req@ffffa06188873100 x1875771511663104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879750.270278:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511663104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879750.270288:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879750.270291:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879750.270295:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879750.270302:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879750.270321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879750.270332:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879750/real 1788879750] req@ffffa06188871500 x1875771511663232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879750.270356:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511663232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879750.270367:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879750.270371:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879750.270375:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879750.270411:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879750.270435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879750.270449:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879750/real 1788879750] req@ffffa06188870700 x1875771511663360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879750.270463:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511663360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879750.270473:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879750.270477:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879750.270481:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879750.909560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879750.909639:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879751.165571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879751.933492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879751.933762:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879752.189522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879752.957519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879752.957616:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879753.213519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879753.981486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879753.981684:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879754.237501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879755.005510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879755.005631:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879755.261472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879755.389624:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879755.389637:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879755.389642:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879755.389656:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4742 00000100:00080000:1.0:1788879755.389671:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879755.389682:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879755.389735:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511663488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879755.389776:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879755.389784:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879755.389787:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879755.389793:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4742 00000100:00080000:1.0:1788879755.389798:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879755.389804:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879755.389819:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879755.389838:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511663616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879755.389844:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511663488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879755.389871:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879755.389879:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879755.389884:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879755.389917:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4742 00000100:00080000:1.0:1788879755.389926:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879755.389935:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879755.389947:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879755.389972:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511663744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879755.389999:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879755.390000:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879755.390013:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879755.390017:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879755.390027:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4742 00000100:00080000:1.0:1788879755.390035:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879755.390042:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879755.390052:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879755.390072:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511663872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879755.390084:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879755/real 1788879755] req@ffffa0618869ad80 x1875771511663488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879755.390090:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879755.390105:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879755.390110:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879755.390124:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4742 00000100:00080000:1.0:1788879755.390135:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879755.390140:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879755.390147:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879755.390157:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511663488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879755.390164:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511664000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879755.390201:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879755.390209:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879755.390219:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511663488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879755.390254:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879755.390290:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879755.390308:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879755/real 1788879755] req@ffffa0618869bb80 x1875771511663616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879755.390325:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511663616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879755.390337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879755.390341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879755.390345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879755.390353:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879755.390374:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879755.390422:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879755/real 1788879755] req@ffffa0618869a680 x1875771511663744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879755.390438:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511663744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879755.390449:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879755.390453:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879755.390457:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879755.390466:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879755.390490:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879755.390504:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879755/real 1788879755] req@ffffa06188698000 x1875771511663872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879755.390518:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511663872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879755.390529:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879755.390532:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879755.390536:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511663872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879755.390543:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879755.390562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879755.390573:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879755/real 1788879755] req@ffffa06188699f80 x1875771511664000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879755.390586:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511664000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879755.390596:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879755.390600:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879755.390603:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879756.029455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879756.029544:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879756.285426:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879757.053530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879757.053605:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879757.309526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879758.077476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879758.077567:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879758.333471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879759.101480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879759.101560:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879759.357453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879760.125533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879760.125647:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879760.381511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879760.509561:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879760.509576:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879760.509582:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879760.509599:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4747 00000100:00080000:3.0:1788879760.509617:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879760.509631:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879760.509692:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511664128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879760.509743:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879760.509752:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879760.509755:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879760.509762:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4747 00000100:00080000:3.0:1788879760.509768:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879760.509775:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879760.509790:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879760.509809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511664128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879760.509816:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511664256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879760.509837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879760.509841:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879760.509843:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879760.509848:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4747 00000100:00080000:3.0:1788879760.509875:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879760.509880:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879760.509894:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879760.509913:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511664384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879760.509926:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879760.509931:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879760.509933:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879760.509938:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4747 00000100:00080000:3.0:1788879760.509944:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879760.509948:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879760.509965:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879760.509976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879760.509984:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511664512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879760.510000:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879760.510005:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879760.510008:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879760.510013:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4747 00000100:00080000:3.0:1788879760.510018:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879760.510023:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879760.510030:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879760.510042:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879760/real 1788879760] req@ffffa06188872a00 x1875771511664128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879760.510050:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511664640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879760.510109:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511664128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879760.510127:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879760.510133:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879760.510139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511664128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879760.510171:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879760.510213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879760.510228:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879760/real 1788879760] req@ffffa06188872300 x1875771511664256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879760.510244:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511664256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879760.510256:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879760.510260:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879760.510264:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879760.510271:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879760.510292:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879760.510303:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879760/real 1788879760] req@ffffa06188873b80 x1875771511664384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879760.510318:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511664384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879760.510328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879760.510332:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879760.510336:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879760.510343:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879760.510362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879760.510373:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879760/real 1788879760] req@ffffa06188872d80 x1875771511664512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879760.510431:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511664512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879760.510442:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879760.510446:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879760.510450:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879760.510457:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879760.510482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879760.510495:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879760/real 1788879760] req@ffffa06188870a80 x1875771511664640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879760.510509:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511664640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879760.510528:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879760.510532:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879760.510536:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879761.149514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879761.149600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879761.405488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879762.173516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879762.173630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879762.429560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879763.197505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879763.197617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879763.453507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879764.221468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879764.221581:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879764.477497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879765.245524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879765.245641:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879765.501473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879765.629536:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879765.629550:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879765.629556:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879765.629571:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4752 00000100:00080000:3.0:1788879765.629588:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879765.629599:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879765.629652:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511664768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879765.629691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879765.629696:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879765.629698:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879765.629704:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4752 00000100:00080000:3.0:1788879765.629709:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879765.629716:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879765.629730:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879765.629747:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511664896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879765.629755:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511664768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879765.629779:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879765.629783:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879765.629786:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879765.629792:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4752 00000100:00080000:3.0:1788879765.629797:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879765.629802:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879765.629809:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879765.629854:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511665024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879765.629868:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879765.629873:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879765.629877:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879765.629881:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4752 00000100:00080000:3.0:1788879765.629886:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879765.629896:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879765.629897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788879765.629907:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879765.629924:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511665152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879765.629947:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879765.629952:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879765.629955:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879765.629960:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879765/real 1788879765] req@ffffa06188870380 x1875771511664768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879765.629966:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4752 00000100:00080000:3.0:1788879765.629976:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879765.629989:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879765.629998:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879765.630010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511664768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879765.630018:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511665280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879765.630056:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879765.630066:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879765.630073:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511664768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879765.630108:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879765.630137:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879765.630152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879765/real 1788879765] req@ffffa06188871880 x1875771511664896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879765.630168:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511664896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879765.630180:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879765.630184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879765.630188:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511664896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879765.630195:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879765.630214:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879765.630225:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879765/real 1788879765] req@ffffa06188873800 x1875771511665024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879765.630239:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511665024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879765.630249:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879765.630252:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879765.630256:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879765.630262:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879765.630281:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879765.630292:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879765/real 1788879765] req@ffffa06188871c00 x1875771511665152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879765.630305:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511665152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879765.630314:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879765.630317:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879765.630322:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879765.630329:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879765.630347:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879765.630357:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879765/real 1788879765] req@ffffa06188870a80 x1875771511665280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879765.630371:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511665280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879765.630411:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879765.630415:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879765.630419:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879766.269528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879766.269614:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879766.525492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879767.293539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879767.293706:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879767.551343:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879768.317464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879768.317590:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879768.573498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879769.341513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879769.341735:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879769.597502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879770.365519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879770.365679:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879770.621487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879770.749622:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879770.749634:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879770.749641:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879770.749655:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4757 00000100:00080000:2.0:1788879770.749669:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879770.749700:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879770.749758:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511665408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879770.749827:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879770.749832:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879770.749835:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879770.749840:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4757 00000100:00080000:2.0:1788879770.749845:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879770.749852:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879770.749866:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879770.749883:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511665536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879770.749900:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511665408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879770.749903:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879770.749916:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879770.749932:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879770.749938:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4757 00000100:00080000:2.0:1788879770.749944:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879770.749949:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879770.749955:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879770.749973:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511665664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879770.749986:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879770.749990:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879770.749992:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879770.749996:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4757 00000100:00080000:2.0:1788879770.750003:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879770.750007:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879770.750013:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879770.750027:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511665792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879770.750039:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879770.750044:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879770.750047:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879770.750052:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4757 00000100:00080000:2.0:1788879770.750056:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879770.750061:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879770.750066:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879770.750087:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511665920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879770.750097:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879770.750176:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879770.750208:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879770.750250:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879771.389502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879771.389602:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879771.645453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879772.413526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879772.413608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879772.669493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879773.437504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879773.437595:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879773.693491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788879773.821728:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788879773.821796:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788879773.821816:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879773.821865:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879773.821878:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879773.821891:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788879773.821901:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879773.821994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879770/real 1788879773] req@ffffa061888adc00 x1875771511665920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879773.822022:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511665920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879773.822035:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879773.822040:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879773.822051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879773.822061:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879770/real 1788879773] req@ffffa061888ae300 x1875771511665536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879773.822074:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511665536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879773.822083:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879773.822087:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879773.822090:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879773.822098:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879770/real 1788879773] req@ffffa061888aea00 x1875771511665664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879773.822114:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511665664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879773.822122:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879773.822126:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879773.822130:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879773.822137:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879770/real 1788879773] req@ffffa061888afb80 x1875771511665792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879773.822150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511665792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879773.822159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879773.822162:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879773.822166:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511665792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879773.822173:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879770/real 1788879773] req@ffffa061888af100 x1875771511665408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879773.822185:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511665408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879773.822193:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879773.822196:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879773.822200:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511665408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879774.461456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879774.461506:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879774.717450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879775.485463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879775.485549:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879775.741543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879775.869531:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879775.869543:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879775.869548:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879775.869561:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4762 00000100:00080000:2.0:1788879775.869576:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879775.869585:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879775.869627:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511666048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879775.869679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879775.869684:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879775.869687:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879775.869692:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4762 00000100:00080000:2.0:1788879775.869697:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879775.869703:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879775.869718:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879775.869741:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511666176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879775.869788:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511666048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879775.869835:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879775.869839:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879775.869841:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879775.869845:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4762 00000100:00080000:2.0:1788879775.869850:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879775.869855:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879775.869861:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879775.869876:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511666304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879775.869904:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879775.869908:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879775.869911:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879775.869914:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4762 00000100:00080000:2.0:1788879775.869919:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879775.869923:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879775.869929:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879775.869933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879775.869948:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511666432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879775.869973:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879775.869978:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879775.869981:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879775.869991:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4762 00000100:00100000:0.0:1788879775.869994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879775/real 1788879775] req@ffffa061888aed80 x1875771511666048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879775.869996:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879775.870008:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879775.870019:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879775.870040:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511666560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879775.870055:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511666048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879775.870085:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879775.870094:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879775.870103:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511666048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879775.870158:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879775.870198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879775.870221:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879775/real 1788879775] req@ffffa061888ac700 x1875771511666176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879775.870244:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511666176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879775.870259:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879775.870264:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879775.870269:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879775.870279:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879775.870314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879775.870335:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879775/real 1788879775] req@ffffa061888ac380 x1875771511666304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879775.870356:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511666304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879775.870370:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879775.870377:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879775.870380:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879775.870391:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879775.870427:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879775.870482:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879775/real 1788879775] req@ffffa061888ad880 x1875771511666432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879775.870504:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511666432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879775.870517:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879775.870522:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879775.870528:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879775.870538:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879775.870597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879775.870619:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879775/real 1788879775] req@ffffa061888ae680 x1875771511666560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879775.870639:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511666560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879775.870658:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879775.870664:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879775.870668:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879776.509532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879776.509681:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879776.765508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879777.533455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879777.533500:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879777.789486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879778.557486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879778.557546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879778.813442:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879779.581559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879779.581706:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879779.837533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879780.605517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879780.605660:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879780.861556:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879780.989553:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879780.989571:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879780.989578:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879780.989596:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4767 00000100:00080000:3.0:1788879780.989614:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879780.989630:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879780.989682:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511666688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879780.989739:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879780.989745:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879780.989748:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879780.989754:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4767 00000100:00080000:3.0:1788879780.989761:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879780.989771:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879780.989791:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879780.989810:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511666688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879780.989825:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511666816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879780.989853:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879780.989860:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879780.989863:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879780.989869:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4767 00000100:00080000:3.0:1788879780.989877:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879780.989883:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879780.989892:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879780.989907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879780.989915:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511666944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879780.989931:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879780.989936:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879780.989938:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879780.989944:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4767 00000100:00080000:3.0:1788879780.989951:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879780.989957:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879780.989966:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879780/real 1788879780] req@ffffa06188872d80 x1875771511666688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879780.989967:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879780.989990:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511667072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879780.990004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511666688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879780.990007:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879780.990023:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879780.990029:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879780.990039:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4767 00000100:00080000:0.0:1788879780.990040:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879780.990046:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879780.990053:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788879780.990060:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879780.990068:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511666688:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879780.990072:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879780.990106:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879780.990109:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511667200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879780.990164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879780.990194:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879780/real 1788879780] req@ffffa06188873b80 x1875771511666816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879780.990220:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511666816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879780.990232:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879780.990237:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879780.990242:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879780.990250:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879780.990278:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879780.990292:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879780/real 1788879780] req@ffffa06188872300 x1875771511666944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879780.990308:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511666944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879780.990319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879780.990323:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879780.990326:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511666944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879780.990334:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879780.990354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879780.990365:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879780/real 1788879780] req@ffffa06188872a00 x1875771511667072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879780.990411:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511667072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879780.990423:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879780.990427:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879780.990431:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879780.990453:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879780.990478:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879780.990491:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879780/real 1788879780] req@ffffa06188870700 x1875771511667200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879780.990506:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511667200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879780.990524:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879780.990528:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879780.990532:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879781.629563:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879781.629654:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879781.885517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879782.653510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879782.653639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879782.909478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879783.677515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879783.677633:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879783.933540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879784.701459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879784.701558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879784.957541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879785.725458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879785.725513:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879785.981526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879786.109607:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879786.109617:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879786.109622:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879786.109633:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4772 00000100:00080000:1.0:1788879786.109645:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879786.109654:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879786.109704:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511667328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879786.109732:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879786.109736:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879786.109738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879786.109745:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4772 00000100:00080000:1.0:1788879786.109750:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879786.109755:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879786.109764:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879786.109769:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511667328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879786.109779:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511667456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879786.109789:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879786.109792:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879786.109793:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879786.109796:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4772 00000100:00080000:1.0:1788879786.109799:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879786.109801:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879786.109804:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879786.109814:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511667584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879786.109820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879786.109823:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879786.109824:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879786.109826:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4772 00000100:00080000:1.0:1788879786.109828:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879786.109830:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879786.109834:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879786.109836:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879786.109844:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511667712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879786.109851:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879786.109854:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879786.109855:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879786.109858:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4772 00000100:00080000:1.0:1788879786.109861:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879786.109864:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879786.109867:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879786.109872:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879786/real 1788879786] req@ffffa06188698380 x1875771511667328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879786.109877:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511667840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879786.109906:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511667328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879786.109916:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879786.109919:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879786.109922:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511667328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879786.109940:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879786.109954:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879786.109962:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879786/real 1788879786] req@ffffa0618869b800 x1875771511667456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879786.109970:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511667456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879786.109977:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879786.109979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879786.109981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879786.109985:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879786.109994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879786.110000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879786/real 1788879786] req@ffffa06188699c00 x1875771511667584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879786.110007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511667584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879786.110013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879786.110015:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879786.110016:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879786.110020:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879786.110031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879786.110037:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879786/real 1788879786] req@ffffa0618869aa00 x1875771511667712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879786.110045:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511667712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879786.110050:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879786.110052:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879786.110055:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879786.110059:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879786.110068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879786.110074:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879786/real 1788879786] req@ffffa0618869b100 x1875771511667840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879786.110081:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511667840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879786.110087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879786.110088:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879786.110090:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511667840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879786.749454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879786.749531:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879787.005459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879787.773459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879787.773519:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879788.029456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879788.797463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879788.797616:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879789.053491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879789.821481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879789.821713:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879790.077489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879790.845505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879790.845622:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879791.101511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879791.229547:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879791.229561:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879791.229566:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879791.229580:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4777 00000100:00080000:3.0:1788879791.229596:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879791.229607:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879791.229679:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511667968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879791.229713:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879791.229719:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879791.229721:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879791.229727:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4777 00000100:00080000:3.0:1788879791.229731:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879791.229736:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879791.229748:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879791.229765:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511668096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879791.229790:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879791.229796:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511667968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879791.229797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879791.229803:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879791.229818:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4777 00000100:00080000:3.0:1788879791.229823:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879791.229827:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879791.229834:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879791.229850:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511668224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879791.229862:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879791.229867:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879791.229870:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879791.229875:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4777 00000100:00080000:3.0:1788879791.229879:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879791.229893:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879791.229901:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879791.229902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879791.229923:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511668352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879791.229937:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879791.229942:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879791.229945:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879791.229951:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4777 00000100:00080000:3.0:1788879791.229956:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879791.229962:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879791/real 1788879791] req@ffffa06188871500 x1875771511667968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879791.229966:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879791.229981:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879791.230007:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511668480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879791.230014:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511667968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879791.230042:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879791.230048:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879791.230056:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511667968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879791.230091:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879791.230119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879791.230135:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879791/real 1788879791] req@ffffa06188873100 x1875771511668096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879791.230151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511668096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879791.230162:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879791.230166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879791.230171:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879791.230178:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879791.230197:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879791.230208:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879791/real 1788879791] req@ffffa06188872680 x1875771511668224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879791.230221:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511668224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879791.230231:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879791.230234:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879791.230238:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879791.230245:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879791.230264:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879791.230275:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879791/real 1788879791] req@ffffa06188873480 x1875771511668352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879791.230290:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511668352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879791.230300:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879791.230303:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879791.230307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879791.230313:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879791.230330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879791.230341:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879791/real 1788879791] req@ffffa06188870700 x1875771511668480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879791.230355:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511668480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879791.230365:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879791.230368:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879791.230371:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879791.869497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879791.869665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879792.125519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879792.893540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879792.893773:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879793.149500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879793.917519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879793.917594:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879794.173479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879794.941531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879794.941728:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879795.197494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879795.965534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879795.965632:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879796.221509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879796.349581:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879796.349593:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879796.349619:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879796.349634:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4783 00000100:00080000:1.0:1788879796.349649:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879796.349658:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879796.349712:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511668608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879796.349776:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879796.349784:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879796.349788:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879796.349797:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4783 00000100:00080000:1.0:1788879796.349805:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879796.349814:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879796.349829:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879796.349846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511668608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879796.349855:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511668736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879796.349882:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879796.349888:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879796.349890:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879796.349894:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4783 00000100:00080000:1.0:1788879796.349899:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879796.349903:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879796.349910:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879796.349924:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511668864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879796.349937:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879796.349942:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879796.349945:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879796.349949:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4783 00000100:00080000:1.0:1788879796.349958:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879796.349962:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879796.349970:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879796.349977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879796.349992:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511668992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879796.350007:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879796.350012:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879796.350015:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879796.350021:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4783 00000100:00080000:1.0:1788879796.350027:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879796.350032:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879796.350041:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879796/real 1788879796] req@ffffa0618869b100 x1875771511668608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879796.350042:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879796.350064:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511669120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879796.350080:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511668608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879796.350099:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879796.350106:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879796.350112:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511668608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879796.350143:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879796.350173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879796.350188:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879796/real 1788879796] req@ffffa0618869aa00 x1875771511668736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879796.350204:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511668736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879796.350216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879796.350220:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879796.350224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879796.350231:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879796.350268:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879796.350282:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879796/real 1788879796] req@ffffa06188699c00 x1875771511668864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879796.350306:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511668864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879796.350316:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879796.350319:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879796.350323:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879796.350330:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879796.350351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879796.350363:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879796/real 1788879796] req@ffffa0618869b800 x1875771511668992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879796.350377:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511668992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879796.350420:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879796.350424:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879796.350427:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511668992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879796.350435:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879796.350459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879796.350473:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879796/real 1788879796] req@ffffa06188698380 x1875771511669120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879796.350488:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511669120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879796.350506:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879796.350510:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879796.350514:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879796.989564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879796.989716:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879797.245538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788879798.013571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879798.013618:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879798.269479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788879799.037553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879799.037695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879799.293555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879800.061528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879800.061563:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879800.317473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879801.085475:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879801.085542:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879801.341475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879801.469550:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879801.469561:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879801.469583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879801.469594:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4788 00000100:00080000:3.0:1788879801.469617:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879801.469624:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879801.469666:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511669248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879801.469694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879801.469697:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879801.469699:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879801.469702:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4788 00000100:00080000:3.0:1788879801.469704:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879801.469709:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879801.469719:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879801.469730:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511669376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879801.469754:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879801.469761:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788879801.469763:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511669248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879801.469764:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879801.469777:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4788 00000100:00080000:3.0:1788879801.469781:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879801.469783:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879801.469786:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879801.469801:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511669504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879801.469807:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879801.469810:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879801.469812:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879801.469814:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4788 00000100:00080000:3.0:1788879801.469816:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879801.469820:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879801.469823:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879801.469830:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511669632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879801.469836:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879801.469838:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879801.469840:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879801.469847:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4788 00000800:00000100:0.0:1788879801.469848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879801.469856:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879801.469862:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879801.469867:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879801.469881:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511669760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879801.469892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879801/real 1788879801] req@ffffa06188872a00 x1875771511669248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879801.469910:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511669248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879801.469920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879801.469924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879801.469927:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511669248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879801.469948:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879801.469964:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879801.469972:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879801/real 1788879801] req@ffffa06188872300 x1875771511669376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879801.469980:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511669376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879801.469991:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879801.469994:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879801.469996:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879801.470000:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879801.470011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879801.470017:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879801/real 1788879801] req@ffffa06188873b80 x1875771511669504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879801.470024:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511669504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879801.470030:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879801.470031:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879801.470033:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879801.470038:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879801.470049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879801.470054:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879801/real 1788879801] req@ffffa06188872d80 x1875771511669632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879801.470062:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511669632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879801.470067:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879801.470069:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879801.470071:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879801.470075:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879801.470085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879801.470091:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879801/real 1788879801] req@ffffa06188870a80 x1875771511669760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879801.470114:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511669760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879801.470120:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879801.470122:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879801.470124:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511669760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879802.109464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879802.109554:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879802.365485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879803.133480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879803.133626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879803.389479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879804.157491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879804.157560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879804.413441:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879805.181517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879805.181716:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879805.437503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879806.205551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879806.205641:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879806.461558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879806.589616:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879806.589628:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879806.589633:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879806.589647:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4793 00000100:00080000:2.0:1788879806.589661:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879806.589671:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879806.589718:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511669888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879806.589754:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879806.589759:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879806.589762:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879806.589767:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4793 00000100:00080000:2.0:1788879806.589771:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879806.589776:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879806.589789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879806.589811:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511669888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879806.589813:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511670016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879806.589844:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879806.589851:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879806.589854:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879806.589862:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4793 00000100:00080000:2.0:1788879806.589870:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879806.589878:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879806.589889:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879806.589909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879806.589919:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511670144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879806.589937:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879806.589942:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879806.589946:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879806.589952:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4793 00000100:00080000:2.0:1788879806.589959:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879806.589965:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879806.589972:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879806.589978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879806/real 1788879806] req@ffffa061888ad180 x1875771511669888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879806.589994:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511670272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879806.590009:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879806.590017:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511669888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879806.590017:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879806.590024:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879806.590037:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4793 00000100:00080000:2.0:1788879806.590044:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879806.590053:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879806.590057:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788879806.590059:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879806.590070:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788879806.590079:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511670400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879806.590089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511669888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879806.590140:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879806.590198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879806.590218:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879806/real 1788879806] req@ffffa061888ace00 x1875771511670016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879806.590235:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511670016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879806.590246:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879806.590250:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879806.590254:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879806.590262:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879806.590285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879806.590296:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879806/real 1788879806] req@ffffa061888ae680 x1875771511670144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879806.590309:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511670144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879806.590319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879806.590322:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879806.590325:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879806.590332:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879806.590351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879806.590362:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879806/real 1788879806] req@ffffa061888ad880 x1875771511670272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879806.590375:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511670272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879806.590414:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879806.590417:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879806.590421:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879806.590428:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879806.590450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879806.590464:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879806/real 1788879806] req@ffffa061888ac380 x1875771511670400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879806.590477:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511670400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879806.590488:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879806.590491:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879806.590495:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879807.229604:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879807.229701:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879807.485555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879808.253546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879808.253583:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879808.509569:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879809.277578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879809.277726:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879809.533534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879810.301620:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879810.301697:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879810.557526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879811.325480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879811.325639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879811.581481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879811.709615:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879811.709631:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879811.709637:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879811.709653:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4798 00000100:00080000:3.0:1788879811.709669:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879811.709678:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879811.709723:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511670528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879811.709760:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879811.709765:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879811.709767:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879811.709772:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4798 00000100:00080000:3.0:1788879811.709777:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879811.709781:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879811.709793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879811.709807:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511670656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879811.709822:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879811.709831:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511670528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879811.709833:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879811.709837:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879811.709852:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4798 00000100:00080000:3.0:1788879811.709857:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879811.709861:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879811.709867:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879811.709882:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511670784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879811.709894:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879811.709898:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879811.709902:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879811.709906:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4798 00000100:00080000:3.0:1788879811.709909:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879811.709913:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879811.709918:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879811.709930:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511670912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879811.709939:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879811.709944:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879811.709946:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879811.709949:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4798 00000100:00080000:3.0:1788879811.709954:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879811.709957:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879811.709966:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879811.709976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879811.709988:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511671040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879811.710067:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879811/real 1788879811] req@ffffa06188871c00 x1875771511670528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879811.710096:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511670528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879811.710112:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879811.710135:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879811.710143:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511670528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879811.710193:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879811.710239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879811.710258:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879811/real 1788879811] req@ffffa06188873800 x1875771511670656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879811.710275:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511670656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879811.710288:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879811.710292:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879811.710297:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879811.710305:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879811.710326:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879811.710338:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879811/real 1788879811] req@ffffa06188871880 x1875771511670784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879811.710352:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511670784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879811.710371:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879811.710375:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879811.710379:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879811.710435:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879811.710461:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879811.710475:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879811/real 1788879811] req@ffffa06188870380 x1875771511670912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879811.710490:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511670912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879811.710544:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879811.710551:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879811.710558:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511670912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879811.710571:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879811.710603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879811.710619:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879811/real 1788879811] req@ffffa06188870a80 x1875771511671040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879811.710634:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511671040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879811.710644:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879811.710648:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879811.710654:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879812.349584:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879812.349694:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879812.605632:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879813.373569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879813.373639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879813.629550:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879814.397567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879814.397695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879814.653546:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879815.421570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879815.421705:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879815.677513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879816.445539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879816.445638:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879816.701581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879816.829446:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879816.829466:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879816.829471:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879816.829481:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4803 00000100:00080000:3.0:1788879816.829491:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879816.829498:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879816.829550:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511671168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879816.829581:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879816.829584:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879816.829586:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879816.829590:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4803 00000100:00080000:3.0:1788879816.829593:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879816.829597:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879816.829607:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879816.829621:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511671296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879816.829623:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511671168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879816.829638:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879816.829640:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879816.829642:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879816.829644:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4803 00000100:00080000:3.0:1788879816.829648:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879816.829651:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879816.829654:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879816.829662:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511671424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879816.829669:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879816.829672:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879816.829673:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879816.829676:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4803 00000100:00080000:3.0:1788879816.829678:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879816.829681:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879816.829684:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879816.829691:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511671552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879816.829697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879816.829702:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879816.829703:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879816.829706:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4803 00000100:00080000:3.0:1788879816.829708:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879816.829711:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879816.829712:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879816.829718:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879816.829727:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511671680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879816.829757:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879816/real 1788879816] req@ffffa06188872d80 x1875771511671168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879816.829781:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511671168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879816.829793:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879816.829797:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879816.829802:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511671168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879816.829828:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879816.829853:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879816.829868:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879816/real 1788879816] req@ffffa06188873b80 x1875771511671296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879816.829882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511671296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879816.829892:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879816.829897:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879816.829903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879816.829910:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879816.829932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879816.829945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879816/real 1788879816] req@ffffa06188872300 x1875771511671424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879816.829958:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511671424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879816.829966:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879816.829969:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879816.829973:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879816.829980:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879816.830003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879816.830015:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879816/real 1788879816] req@ffffa06188872a00 x1875771511671552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879816.830028:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511671552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879816.830038:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879816.830042:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879816.830055:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879816.830062:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879816.830082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879816.830095:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879816/real 1788879816] req@ffffa06188870700 x1875771511671680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879816.830109:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511671680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879816.830118:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879816.830122:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879816.830125:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879817.469558:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879817.469611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879817.725466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879818.493560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879818.493672:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879818.749523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879819.517543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879819.517719:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879819.773538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879820.541530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879820.541670:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879820.797492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879821.565562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879821.565709:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879821.821547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879821.949659:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879821.949675:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879821.949681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879821.949695:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4808 00000100:00080000:2.0:1788879821.949712:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879821.949726:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879821.949787:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511671808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879821.949827:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879821.949835:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879821.949838:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879821.949848:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4808 00000100:00080000:2.0:1788879821.949857:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879821.949867:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879821.949885:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879821.949894:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511671808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879821.949917:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511671936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879821.949936:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879821.949943:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879821.949947:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879821.949955:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4808 00000100:00080000:2.0:1788879821.949966:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879821.949975:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879821.949985:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879821.950009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879821.950014:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511672064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879821.950034:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879821.950039:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879821.950042:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879821.950047:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4808 00000100:00080000:2.0:1788879821.950053:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879821.950059:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879821.950067:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879821.950078:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879821/real 1788879821] req@ffffa061888ac700 x1875771511671808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879821.950092:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511672192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879821.950126:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879821.950135:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788879821.950137:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511671808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879821.950141:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879821.950159:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4808 00000100:00080000:2.0:1788879821.950173:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879821.950178:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879821.950182:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879821.950193:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:2.0:1788879821.950195:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879821.950208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511671808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879821.950220:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511672320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879821.950259:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879821.950311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879821.950338:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879821/real 1788879821] req@ffffa061888aed80 x1875771511671936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879821.950364:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511671936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879821.950441:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879821.950451:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879821.950458:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511671936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879821.950471:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879821.950511:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879821.950536:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879821/real 1788879821] req@ffffa061888af100 x1875771511672064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879821.950563:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511672064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879821.950579:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879821.950586:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879821.950593:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879821.950606:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879821.950643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879821.950662:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879821/real 1788879821] req@ffffa061888afb80 x1875771511672192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879821.950684:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511672192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879821.950702:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879821.950708:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879821.950714:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879821.950726:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879821.950763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879821.950784:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879821/real 1788879821] req@ffffa061888aea00 x1875771511672320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879821.950805:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511672320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879821.950821:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879821.950827:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879821.950834:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879822.589528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879822.589600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879822.845537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879823.613522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879823.613693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879823.869585:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879824.637505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879824.637643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879824.893579:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879825.661548:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879825.661627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879825.917454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879826.685506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879826.685629:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879826.941518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879827.069487:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879827.069497:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879827.069502:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879827.069513:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4813 00000100:00080000:2.0:1788879827.069525:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879827.069532:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879827.069570:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511672448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879827.069595:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879827.069598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879827.069601:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879827.069604:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4813 00000100:00080000:2.0:1788879827.069608:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879827.069612:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879827.069622:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879827.069634:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511672448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879827.069643:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511672576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879827.069653:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879827.069656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879827.069658:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879827.069660:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4813 00000100:00080000:2.0:1788879827.069663:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879827.069666:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879827.069671:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879827.069681:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511672704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879827.069688:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879827.069692:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879827.069693:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879827.069696:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4813 00000100:00080000:2.0:1788879827.069699:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879827.069702:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879827.069707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788879827.069716:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879827.069729:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511672832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879827.069738:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879827.069741:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879827.069742:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879827.069747:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4813 00000100:00100000:0.0:1788879827.069748:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879827/real 1788879827] req@ffffa061888ae300 x1875771511672448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879827.069750:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879827.069759:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879827.069767:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879827.069779:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511672448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879827.069781:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511672960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879827.069816:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879827.069822:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879827.069826:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511672448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879827.069849:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879827.069867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879827.069876:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879827/real 1788879827] req@ffffa061888adc00 x1875771511672576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879827.069886:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511672576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879827.069892:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879827.069895:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879827.069898:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879827.069902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879827.069913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879827.069920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879827/real 1788879827] req@ffffa061888ad500 x1875771511672704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879827.069928:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511672704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879827.069935:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879827.069937:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879827.069939:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879827.069944:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879827.069956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879827.069962:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879827/real 1788879827] req@ffffa061888ac000 x1875771511672832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879827.069971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511672832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879827.069977:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879827.069979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879827.069981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879827.069985:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879827.069996:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879827.070002:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879827/real 1788879827] req@ffffa061888adf80 x1875771511672960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879827.070011:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511672960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879827.070017:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879827.070019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879827.070021:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511672960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879827.709525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879827.709667:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879827.965504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879828.733513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879828.733629:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879828.989570:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879829.757515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879829.757597:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879830.013568:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879830.781573:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879830.781681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879831.037496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879831.805532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879831.805647:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879832.061510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879832.189506:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879832.189520:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879832.189526:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879832.189542:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4818 00000100:00080000:2.0:1788879832.189560:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879832.189571:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879832.189623:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511673088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879832.189662:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879832.189668:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879832.189673:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879832.189682:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4818 00000100:00080000:2.0:1788879832.189690:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879832.189699:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879832.189716:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879832.189719:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511673088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879832.189751:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511673216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879832.189784:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879832.189792:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879832.189796:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879832.189802:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4818 00000100:00080000:2.0:1788879832.189809:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879832.189815:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879832.189823:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879832.189829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879832.189852:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511673344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879832.189868:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879832.189873:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879832.189876:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879832.189884:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4818 00000100:00100000:0.0:1788879832.189885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879832/real 1788879832] req@ffffa061888aca80 x1875771511673088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879832.189891:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879832.189904:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879832.189918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879832.189933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511673088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879832.189941:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511673472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879832.189976:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879832.189978:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879832.189990:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879832.189995:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879832.189997:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788879832.190004:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4818 00000100:00080000:2.0:1788879832.190011:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879832.190017:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511673088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879832.190020:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879832.190031:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879832.190047:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511673600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879832.190075:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879832.190119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879832.190137:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879832/real 1788879832] req@ffffa061888af480 x1875771511673216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879832.190153:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511673216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879832.190165:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879832.190170:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879832.190174:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879832.190182:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879832.190202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879832.190214:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879832/real 1788879832] req@ffffa061888adf80 x1875771511673344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879832.190229:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511673344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879832.190239:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879832.190243:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879832.190246:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879832.190253:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879832.190272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879832.190283:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879832/real 1788879832] req@ffffa061888ac000 x1875771511673472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879832.190297:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511673472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879832.190307:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879832.190311:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879832.190314:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879832.190320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879832.190338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879832.190376:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879832/real 1788879832] req@ffffa061888ad500 x1875771511673600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879832.190425:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511673600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879832.190445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879832.190449:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879832.190453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879832.829526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879832.829611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879833.085507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879833.853566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879833.853671:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879834.109520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879834.877505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879834.877644:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879835.133439:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879835.901495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879835.901567:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879836.157447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879836.925532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879836.925624:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879837.181507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879837.309568:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879837.309583:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879837.309590:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879837.309606:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4824 00000100:00080000:3.0:1788879837.309622:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879837.309634:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879837.309685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511673728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879837.309720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879837.309726:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879837.309728:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879837.309734:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4824 00000100:00080000:3.0:1788879837.309739:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879837.309745:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879837.309759:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879837.309778:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511673856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879837.309787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511673728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879837.309809:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879837.309815:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879837.309818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879837.309823:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4824 00000100:00080000:3.0:1788879837.309826:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879837.309832:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879837.309839:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879837.309857:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511673984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879837.309870:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879837.309875:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879837.309878:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879837.309882:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4824 00000100:00080000:3.0:1788879837.309887:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879837.309891:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879837.309897:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879837.309913:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511674112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879837.309924:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879837.309929:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879837.309932:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879837.309937:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4824 00000100:00080000:3.0:1788879837.309942:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879837.309959:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879837.309966:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879837.309986:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788879837.309988:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511674240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879837.310071:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879837.310107:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879837.310139:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879837.949535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879837.949654:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879838.205495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879838.973485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879838.973540:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879839.229442:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879839.997472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879839.997547:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879840.253504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879840.381617:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879840.381655:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879840.381668:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879840.381699:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879840.381706:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879840.381713:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879840.381719:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879840.381742:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879837/real 1788879840] req@ffffa06188870700 x1875771511674240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879840.381771:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511674240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879840.381785:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879840.381789:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879840.381794:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879840.381800:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879837/real 1788879840] req@ffffa06188872680 x1875771511673856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879840.381810:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511673856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879840.381818:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879840.381821:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879840.381825:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879840.381833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879837/real 1788879840] req@ffffa06188873100 x1875771511673984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879840.381843:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511673984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879840.381851:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879840.381854:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879840.381871:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511673984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879840.381883:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879837/real 1788879840] req@ffffa06188871500 x1875771511674112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879840.381895:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511674112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879840.381904:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879840.381907:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879840.381911:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879840.381919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879837/real 1788879840] req@ffffa06188873480 x1875771511673728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879840.381931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511673728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879840.381938:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879840.381941:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879840.381945:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511673728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879841.021476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879841.021521:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879841.277433:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879842.045473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879842.045549:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879842.301498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879842.429523:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879842.429531:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879842.429534:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879842.429543:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4829 00000100:00080000:2.0:1788879842.429551:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879842.429557:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879842.429586:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511674368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879842.429608:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879842.429610:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879842.429612:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879842.429615:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4829 00000100:00080000:2.0:1788879842.429617:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879842.429620:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879842.429628:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879842.429642:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511674496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879842.429653:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879842.429655:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511674368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879842.429657:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879842.429662:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879842.429668:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4829 00000100:00080000:2.0:1788879842.429671:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879842.429675:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879842.429680:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879842.429694:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511674624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879842.429704:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879842.429708:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879842.429710:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879842.429714:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4829 00000100:00080000:2.0:1788879842.429718:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879842.429721:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879842.429727:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879842.429730:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879842.429742:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511674752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879842.429752:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879842.429756:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879842.429758:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879842.429762:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4829 00000100:00080000:2.0:1788879842.429766:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879842.429770:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879842.429775:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879842/real 1788879842] req@ffffa061888adc00 x1875771511674368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879842.429777:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879842.429795:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511674880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879842.429803:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511674368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879842.429818:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879842.429823:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879842.429828:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511674368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879842.429864:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879842.429890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879842.429904:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879842/real 1788879842] req@ffffa061888ae300 x1875771511674496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879842.429919:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511674496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879842.429929:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879842.429932:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879842.429936:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879842.429943:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879842.429965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879842.429977:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879842/real 1788879842] req@ffffa061888aea00 x1875771511674624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879842.429991:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511674624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879842.429999:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879842.430003:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879842.430006:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879842.430013:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879842.430035:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879842.430046:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879842/real 1788879842] req@ffffa061888afb80 x1875771511674752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879842.430058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511674752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879842.430074:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879842.430078:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879842.430082:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879842.430088:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879842.430109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879842.430120:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879842/real 1788879842] req@ffffa061888af100 x1875771511674880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879842.430138:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511674880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879842.430146:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879842.430150:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879842.430153:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511674880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879843.069489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879843.069601:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879843.325450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879844.093469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879844.093544:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879844.349446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879845.117477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879845.117562:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879845.373471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879846.141501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879846.141642:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879846.397449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879847.165472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879847.165491:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879847.421454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879847.549506:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879847.549520:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879847.549526:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879847.549541:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4834 00000100:00080000:3.0:1788879847.549557:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879847.549568:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879847.549617:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511675008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879847.549656:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879847.549661:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879847.549664:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879847.549669:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4834 00000100:00080000:3.0:1788879847.549674:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879847.549679:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879847.549693:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879847.549718:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511675008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879847.549719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511675136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879847.549745:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879847.549750:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879847.549752:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879847.549758:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4834 00000100:00080000:3.0:1788879847.549762:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879847.549766:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879847.549774:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879847.549789:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511675264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879847.549809:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879847.549814:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879847.549818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879847.549822:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4834 00000100:00080000:3.0:1788879847.549826:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879847.549829:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879847.549836:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879847.549843:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879847.549856:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511675392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879847.549883:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879847.549889:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879847.549892:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879847.549898:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4834 00000100:00080000:3.0:1788879847.549904:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879847.549928:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879847.549929:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879847/real 1788879847] req@ffffa06188872a00 x1875771511675008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879847.549932:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879847.549949:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511675520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879847.549956:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511675008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879847.549969:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879847.549974:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879847.549979:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511675008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879847.550000:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879847.550049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879847.550066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879847/real 1788879847] req@ffffa06188872300 x1875771511675136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879847.550082:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511675136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879847.550093:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879847.550098:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879847.550102:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879847.550109:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879847.550129:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879847.550141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879847/real 1788879847] req@ffffa06188873b80 x1875771511675264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879847.550155:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511675264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879847.550165:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879847.550168:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879847.550172:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879847.550180:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879847.550215:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879847.550229:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879847/real 1788879847] req@ffffa06188872d80 x1875771511675392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879847.550274:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511675392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879847.550286:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879847.550306:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879847.550309:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879847.550314:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879847.550330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879847.550337:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879847/real 1788879847] req@ffffa06188870a80 x1875771511675520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879847.550346:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511675520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879847.550352:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879847.550354:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879847.550356:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879848.189506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879848.189599:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879848.445465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879849.213523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879849.213659:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879849.469506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879850.237508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879850.237669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879850.493559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879851.261556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879851.261695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879851.517513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879852.285467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879852.285566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879852.541510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879852.669483:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879852.669491:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879852.669495:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879852.669505:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4839 00000100:00080000:3.0:1788879852.669515:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879852.669522:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879852.669555:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511675648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879852.669577:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879852.669579:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879852.669580:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879852.669583:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4839 00000100:00080000:3.0:1788879852.669585:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879852.669587:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879852.669595:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879852.669604:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511675776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879852.669613:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879852.669615:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511675648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879852.669618:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879852.669622:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879852.669626:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4839 00000100:00080000:3.0:1788879852.669627:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879852.669630:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879852.669633:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879852.669643:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511675904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879852.669649:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879852.669652:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879852.669653:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879852.669655:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4839 00000100:00080000:3.0:1788879852.669659:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879852.669661:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879852.669664:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879852.669671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511676032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879852.669677:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879852.669680:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879852.669681:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879852.669685:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879852.669689:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4839 00000100:00080000:3.0:1788879852.669693:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879852.669696:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879852.669700:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879852.669710:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511676160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879852.669720:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879852/real 1788879852] req@ffffa06188870380 x1875771511675648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879852.669735:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511675648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879852.669743:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879852.669746:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879852.669749:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511675648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879852.669768:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879852.669785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879852.669793:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879852/real 1788879852] req@ffffa06188871880 x1875771511675776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879852.669802:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511675776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879852.669808:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879852.669810:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879852.669813:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879852.669816:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879852.669825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879852.669831:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879852/real 1788879852] req@ffffa06188873800 x1875771511675904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879852.669840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511675904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879852.669845:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879852.669848:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879852.669850:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511675904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879852.669854:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879852.669865:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879852.669871:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879852/real 1788879852] req@ffffa06188871c00 x1875771511676032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879852.669879:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511676032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879852.669884:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879852.669886:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879852.669887:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879852.669891:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879852.669900:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879852.669905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879852/real 1788879852] req@ffffa06188870a80 x1875771511676160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879852.669912:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511676160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879852.669922:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879852.669924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879852.669926:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879853.309483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879853.309547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879853.565529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879854.333480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879854.333562:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879854.589473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879855.357501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879855.357574:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879855.613611:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879856.381484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879856.381563:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879856.637504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879857.405493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879857.405557:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879857.661535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879857.789519:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879857.789533:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879857.789539:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879857.789554:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4844 00000100:00080000:3.0:1788879857.789570:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879857.789580:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879857.789630:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511676288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879857.789669:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879857.789673:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879857.789676:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879857.789681:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4844 00000100:00080000:3.0:1788879857.789686:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879857.789691:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879857.789705:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879857.789723:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511676416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879857.789733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511676288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879857.789758:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879857.789763:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879857.789765:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879857.789773:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4844 00000100:00080000:3.0:1788879857.789777:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879857.789781:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879857.789789:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879857.789807:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511676544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879857.789820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879857.789826:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879857.789829:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879857.789833:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4844 00000100:00080000:3.0:1788879857.789838:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879857.789844:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879857.789856:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788879857.789857:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879857.789877:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511676672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879857.789891:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879857.789895:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879857.789897:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879857.789903:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4844 00000100:00080000:3.0:1788879857.789909:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879857.789914:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879857.789921:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879857.789928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879857/real 1788879857] req@ffffa06188872d80 x1875771511676288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879857.789947:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511676800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879857.789965:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511676288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879857.789986:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879857.789993:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879857.789999:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511676288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879857.790032:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879857.790064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879857.790080:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879857/real 1788879857] req@ffffa06188873b80 x1875771511676416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879857.790095:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511676416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879857.790107:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879857.790111:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879857.790115:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879857.790122:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879857.790141:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879857.790152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879857/real 1788879857] req@ffffa06188872300 x1875771511676544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879857.790166:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511676544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879857.790219:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879857.790223:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879857.790225:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879857.790231:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879857.790252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879857.790262:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879857/real 1788879857] req@ffffa06188872a00 x1875771511676672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879857.790271:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511676672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879857.790278:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879857.790281:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879857.790284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879857.790289:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879857.790302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879857.790309:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879857/real 1788879857] req@ffffa06188873480 x1875771511676800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879857.790319:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511676800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879857.790326:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879857.790328:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879857.790330:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511676800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879858.429517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879858.429598:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879858.685450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879859.453473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879859.453540:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879859.709485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879860.477489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879860.477612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879860.733498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879861.501518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879861.501672:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879861.757524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879862.525486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879862.525569:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879862.781487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879862.909663:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879862.909677:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879862.909685:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879862.909702:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4849 00000100:00080000:2.0:1788879862.909717:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879862.909727:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879862.909792:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511676928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879862.909826:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879862.909831:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879862.909833:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879862.909849:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4849 00000100:00080000:2.0:1788879862.909853:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879862.909859:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879862.909872:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879862.909892:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511677056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879862.909896:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511676928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879862.909923:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879862.909928:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879862.909930:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879862.909935:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4849 00000100:00080000:2.0:1788879862.909939:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879862.909943:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879862.909950:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879862.909966:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511677184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879862.909977:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879862.909981:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879862.909984:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879862.909988:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4849 00000100:00080000:2.0:1788879862.909993:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879862.909996:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879862.910013:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879862.910033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879862.910039:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511677312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879862.910054:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879862.910059:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879862.910062:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879862.910067:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4849 00000100:00080000:2.0:1788879862.910072:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879862.910088:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879862.910097:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879862.910105:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879862/real 1788879862] req@ffffa061888aed80 x1875771511676928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879862.910114:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511677440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879862.910194:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511676928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879862.910212:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879862.910221:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879862.910232:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511676928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879862.910272:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879862.910320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879862.910347:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879862/real 1788879862] req@ffffa061888ac700 x1875771511677056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879862.910368:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511677056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879862.910414:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879862.910419:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879862.910424:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879862.910432:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879862.910462:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879862.910477:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879862/real 1788879862] req@ffffa061888ac380 x1875771511677184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879862.910491:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511677184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879862.910501:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879862.910505:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879862.910509:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879862.910515:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879862.910534:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879862.910546:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879862/real 1788879862] req@ffffa061888ad880 x1875771511677312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879862.910560:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511677312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879862.910569:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879862.910573:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879862.910576:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879862.910583:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879862.910600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879862.910611:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879862/real 1788879862] req@ffffa061888ae680 x1875771511677440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879862.910624:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511677440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879862.910634:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879862.910637:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879862.910640:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879863.549540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879863.549594:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879863.805498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879864.573519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879864.573572:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879864.829473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879865.597516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879865.597609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879865.853515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879866.621501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879866.621627:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879866.877483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879867.645517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879867.645623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879867.901525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879868.029553:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879868.029564:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879868.029568:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879868.029579:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4854 00000100:00080000:3.0:1788879868.029591:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879868.029598:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879868.029641:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511677568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879868.029667:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879868.029672:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879868.029674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879868.029677:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4854 00000100:00080000:3.0:1788879868.029680:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879868.029683:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879868.029692:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879868.029710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511677696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879868.029713:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511677568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879868.029730:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879868.029734:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879868.029736:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879868.029739:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4854 00000100:00080000:3.0:1788879868.029741:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879868.029743:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879868.029746:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879868.029754:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511677824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879868.029760:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879868.029762:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879868.029764:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879868.029766:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4854 00000100:00080000:3.0:1788879868.029769:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879868.029773:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879868.029777:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879868.029784:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511677952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879868.029792:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879868.029794:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879868.029801:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879868.029802:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879868.029814:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4854 00000100:00080000:3.0:1788879868.029819:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879868.029822:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879868.029827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788879868.029841:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511678080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879868.029853:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879868/real 1788879868] req@ffffa06188871500 x1875771511677568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879868.029875:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511677568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879868.029884:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879868.029888:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879868.029891:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511677568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879868.029912:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879868.029929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879868.029938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879868/real 1788879868] req@ffffa06188873100 x1875771511677696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879868.029946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511677696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879868.029953:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879868.029955:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879868.029957:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879868.029962:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879868.029972:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879868.029978:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879868/real 1788879868] req@ffffa06188872680 x1875771511677824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879868.029986:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511677824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879868.030001:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879868.030005:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879868.030009:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879868.030016:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879868.030039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879868.030051:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879868/real 1788879868] req@ffffa06188870700 x1875771511677952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879868.030064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511677952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879868.030070:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879868.030072:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879868.030075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511677952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879868.030079:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879868.030111:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879868.030121:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879868/real 1788879868] req@ffffa06188873480 x1875771511678080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879868.030130:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511678080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879868.030135:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879868.030138:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879868.030141:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879868.669551:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879868.669621:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879868.925471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879869.693496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879869.693705:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879869.949483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879870.717477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879870.717536:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879870.973559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879871.741471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879871.741586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879871.997457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879872.765533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879872.765667:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879873.021488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879873.149590:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879873.149603:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879873.149609:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879873.149622:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4859 00000100:00080000:1.0:1788879873.149636:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879873.149647:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879873.149693:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511678208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879873.149731:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879873.149736:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879873.149738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879873.149744:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4859 00000100:00080000:1.0:1788879873.149748:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879873.149752:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879873.149776:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879873.149793:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511678336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879873.149808:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879873.149814:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788879873.149820:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511678208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879873.149820:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879873.149834:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4859 00000100:00080000:1.0:1788879873.149840:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879873.149844:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879873.149851:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879873.149869:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511678464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879873.149882:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879873.149887:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879873.149891:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879873.149896:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4859 00000100:00080000:1.0:1788879873.149914:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879873.149919:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879873.149928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879873.149930:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879873.149951:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511678592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879873.149965:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879873.149970:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879873.149972:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879873.149978:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4859 00000100:00100000:0.0:1788879873.149985:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879873/real 1788879873] req@ffffa06188699f80 x1875771511678208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879873.149987:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879873.150002:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879873.150017:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879873.150033:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511678208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879873.150035:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511678720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879873.150100:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879873.150111:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879873.150130:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511678208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879873.150163:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879873.150196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879873.150225:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879873/real 1788879873] req@ffffa06188698000 x1875771511678336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879873.150240:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511678336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879873.150252:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879873.150257:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879873.150262:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879873.150269:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879873.150289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879873.150300:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879873/real 1788879873] req@ffffa0618869a680 x1875771511678464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879873.150314:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511678464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879873.150324:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879873.150327:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879873.150331:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879873.150338:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879873.150356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879873.150367:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879873/real 1788879873] req@ffffa0618869bb80 x1875771511678592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879873.150412:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511678592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879873.150424:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879873.150428:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879873.150432:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879873.150440:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879873.150463:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879873.150477:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879873/real 1788879873] req@ffffa0618869ad80 x1875771511678720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879873.150492:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511678720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879873.150507:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879873.150511:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879873.150515:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879873.789557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879873.789629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879874.045458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879874.813490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879874.813624:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879875.069519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879875.837533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879875.837642:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879876.093460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879876.861505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788879876.861590:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879877.117475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879877.885473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879877.885534:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879878.141448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879878.269502:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879878.269516:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879878.269524:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879878.269539:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4864 00000100:00080000:1.0:1788879878.269556:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879878.269569:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879878.269627:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511678848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879878.269665:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879878.269673:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879878.269677:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879878.269686:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4864 00000100:00080000:1.0:1788879878.269694:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879878.269703:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879878.269730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879878.269735:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511678848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879878.269754:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511678976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879878.269773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879878.269779:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879878.269782:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879878.269788:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4864 00000100:00080000:1.0:1788879878.269822:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879878.269830:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879878.269837:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879878.269839:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879878.269858:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511679104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879878.269888:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879878.269894:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879878.269898:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879878.269905:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4864 00000100:00100000:0.0:1788879878.269908:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879878/real 1788879878] req@ffffa0618869ad80 x1875771511678848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879878.269911:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879878.269919:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879878.269928:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879878.269947:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511679232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879878.269958:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511678848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879878.269964:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879878.269971:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879878.269974:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879878.269980:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4864 00000100:00080000:0.0:1788879878.269986:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879878.269987:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788879878.269994:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788879878.269994:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879878.270002:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511678848:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879878.270003:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879878.270021:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511679360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879878.270071:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879878.270118:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879878.270149:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879878/real 1788879878] req@ffffa0618869bb80 x1875771511678976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879878.270167:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511678976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879878.270178:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879878.270182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879878.270187:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511678976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879878.270195:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879878.270221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879878.270234:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879878/real 1788879878] req@ffffa0618869a680 x1875771511679104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879878.270248:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511679104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879878.270258:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879878.270262:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879878.270266:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879878.270273:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879878.270291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879878.270302:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879878/real 1788879878] req@ffffa06188698000 x1875771511679232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879878.270316:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511679232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879878.270327:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879878.270330:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879878.270334:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879878.270364:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879878.270416:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879878.270431:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879878/real 1788879878] req@ffffa06188699f80 x1875771511679360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879878.270446:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511679360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879878.270456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879878.270460:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879878.270463:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879878.909470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879878.909554:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879879.165453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879879.933464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879879.933546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879880.189502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879880.957510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879880.957617:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879881.213489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879881.981536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879881.981688:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879882.237590:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879883.005571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879883.005624:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879883.261543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879883.389581:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879883.389593:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879883.389598:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879883.389612:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4870 00000100:00080000:3.0:1788879883.389627:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879883.389636:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879883.389691:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511679488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879883.389741:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879883.389750:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879883.389754:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879883.389765:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4870 00000100:00080000:3.0:1788879883.389775:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879883.389785:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879883.389799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511679488:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788879883.389803:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879883.389834:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511679616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879883.389851:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879883.389856:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879883.389859:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879883.389864:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4870 00000100:00080000:3.0:1788879883.389870:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879883.389875:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879883.389882:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879883.389898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788879883.389900:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511679744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879883.389917:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879883.389922:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879883.389925:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879883.389930:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4870 00000100:00080000:3.0:1788879883.389935:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879883.389941:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879883.389948:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879883.389958:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879883/real 1788879883] req@ffffa06188872a00 x1875771511679488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879883.389964:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511679872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879883.390036:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879883.390041:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879883.390044:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879883.390051:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511679488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879883.390056:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4870 00000100:00080000:3.0:1788879883.390067:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879883.390080:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879883.390086:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788879883.390088:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879883.390104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788879883.390115:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511680000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879883.390121:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511679488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879883.390180:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879883.390228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879883.390256:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879883/real 1788879883] req@ffffa06188872300 x1875771511679616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879883.390285:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511679616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879883.390304:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879883.390309:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879883.390313:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879883.390322:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879883.390352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879883.390367:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879883/real 1788879883] req@ffffa06188873b80 x1875771511679744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879883.390415:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511679744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879883.390427:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879883.390431:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879883.390435:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879883.390444:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879883.390470:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879883.390484:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879883/real 1788879883] req@ffffa06188872d80 x1875771511679872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879883.390498:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511679872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879883.390508:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879883.390512:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879883.390516:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511679872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879883.390523:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879883.390541:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879883.390552:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879883/real 1788879883] req@ffffa06188870a80 x1875771511680000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879883.390565:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511680000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879883.390575:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879883.390579:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879883.390583:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879884.029516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879884.029545:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879884.285541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879885.053483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879885.053543:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879885.309466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879886.077473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879886.077596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879886.333501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879887.101466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879887.101563:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879887.357495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879888.125476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879888.125580:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879888.381455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879888.509561:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879888.509567:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879888.509570:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879888.509579:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4875 00000100:00080000:2.0:1788879888.509589:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879888.509594:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879888.509629:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511680128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879888.509650:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879888.509653:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879888.509654:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879888.509657:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4875 00000100:00080000:2.0:1788879888.509659:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879888.509662:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879888.509672:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879888.509682:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511680128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879888.509683:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511680256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879888.509696:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879888.509699:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879888.509700:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879888.509702:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4875 00000100:00080000:2.0:1788879888.509704:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879888.509706:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879888.509710:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879888.509721:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511680384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879888.509728:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879888.509731:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879888.509732:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879888.509735:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4875 00000100:00080000:2.0:1788879888.509738:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879888.509741:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879888.509744:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879888.509749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879888.509754:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511680512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879888.509761:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879888.509764:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879888.509765:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879888.509768:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4875 00000100:00080000:2.0:1788879888.509770:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879888.509772:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879888.509775:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879888.509783:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511680640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879888.509787:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879888/real 1788879888] req@ffffa061888ace00 x1875771511680128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879888.509805:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511680128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879888.509814:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879888.509817:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879888.509820:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511680128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879888.509837:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879888.509851:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879888.509859:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879888/real 1788879888] req@ffffa061888ad180 x1875771511680256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879888.509866:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511680256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879888.509872:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879888.509874:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879888.509876:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879888.509879:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879888.509890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879888.509895:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879888/real 1788879888] req@ffffa061888ae680 x1875771511680384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879888.509903:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511680384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879888.509909:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879888.509910:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879888.509912:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879888.509915:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879888.509925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879888.509930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879888/real 1788879888] req@ffffa061888ad880 x1875771511680512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879888.509937:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511680512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879888.509943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879888.509944:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879888.509946:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879888.509949:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879888.509976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879888.509984:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879888/real 1788879888] req@ffffa061888ac380 x1875771511680640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879888.509990:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511680640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879888.510000:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879888.510002:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879888.510004:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879889.149493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879889.149510:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879889.405469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879890.173503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879890.173665:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879890.429584:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879891.197555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879891.197647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879891.453482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879892.221446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879892.221507:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879892.477466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879893.245497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879893.245626:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879893.501528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879893.629518:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879893.629530:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879893.629537:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879893.629550:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4880 00000100:00080000:2.0:1788879893.629566:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879893.629579:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879893.629630:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511680768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879893.629665:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879893.629671:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879893.629673:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879893.629678:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4880 00000100:00080000:2.0:1788879893.629683:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879893.629687:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879893.629700:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879893.629715:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511680896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879893.629728:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879893.629733:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879893.629737:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879893.629744:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4880 00000100:00080000:2.0:1788879893.629753:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879893.629760:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879893.629770:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879893.629790:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511680768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879893.629799:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511681024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879893.629826:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879893.629834:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879893.629839:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879893.629847:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4880 00000100:00080000:2.0:1788879893.629854:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879893.629860:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879893.629868:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879893.629891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879893.629893:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511681152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879893.629910:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879893.629939:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879893.629942:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879893.629949:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4880 00000100:00080000:2.0:1788879893.629955:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879893.629961:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879893.629969:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879893.629985:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879893/real 1788879893] req@ffffa061888ac700 x1875771511680768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879893.629990:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511681280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879893.630066:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511680768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879893.630082:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879893.630088:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879893.630093:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511680768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879893.630103:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879893.630136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879893.630152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879893/real 1788879893] req@ffffa061888aed80 x1875771511680896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879893.630168:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511680896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879893.630179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879893.630183:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879893.630188:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511680896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879893.630217:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879893.630237:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879893.630255:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879893/real 1788879893] req@ffffa061888af100 x1875771511681024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879893.630278:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511681024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879893.630294:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879893.630300:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879893.630307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879893.630319:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879893.630359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879893.630414:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879893/real 1788879893] req@ffffa061888afb80 x1875771511681152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879893.630433:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511681152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879893.630444:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879893.630448:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879893.630453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879893.630461:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879893.630489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879893.630503:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879893/real 1788879893] req@ffffa061888aea00 x1875771511681280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879893.630517:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511681280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879893.630527:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879893.630531:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879893.630534:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879894.269521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879894.269721:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879894.525523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879895.293527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879895.293666:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879895.549457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879896.317495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879896.317579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879896.573530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879897.341493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879897.341660:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879897.597510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879898.365507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879898.365671:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879898.621575:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879898.749612:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879898.749626:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879898.749632:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879898.749648:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4885 00000100:00080000:1.0:1788879898.749665:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879898.749676:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879898.749735:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511681408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879898.749780:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879898.749785:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879898.749788:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879898.749794:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4885 00000100:00080000:1.0:1788879898.749800:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879898.749810:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879898.749827:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879898.749860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511681408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879898.749860:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511681536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879898.749919:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879898.749925:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879898.749928:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879898.749935:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4885 00000100:00080000:1.0:1788879898.749941:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879898.749946:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879898.749953:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879898.749969:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511681664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879898.749982:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879898.749986:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879898.749989:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879898.749996:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4885 00000100:00080000:1.0:1788879898.750001:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879898.750005:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879898.750012:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879898.750026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879898.750037:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511681792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879898.750056:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879898.750062:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879898.750065:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879898.750075:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4885 00000100:00080000:1.0:1788879898.750083:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879898.750089:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879898.750095:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879898.750099:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879898/real 1788879898] req@ffffa06188698380 x1875771511681408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879898.750116:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511681920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879898.750153:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511681408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879898.750172:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879898.750179:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879898.750187:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511681408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879898.750221:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879898.750252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879898.750267:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879898/real 1788879898] req@ffffa0618869b800 x1875771511681536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879898.750283:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511681536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879898.750294:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879898.750298:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879898.750302:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879898.750310:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879898.750329:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879898.750341:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879898/real 1788879898] req@ffffa06188699c00 x1875771511681664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879898.750354:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511681664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879898.750364:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879898.750368:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879898.750371:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879898.750379:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879898.750438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879898.750454:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879898/real 1788879898] req@ffffa0618869aa00 x1875771511681792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879898.750469:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511681792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879898.750479:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879898.750483:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879898.750487:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879898.750494:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879898.750513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879898.750524:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879898/real 1788879898] req@ffffa0618869b100 x1875771511681920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879898.750538:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511681920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879898.750547:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879898.750551:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879898.750554:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511681920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879899.389473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879899.389604:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879899.645509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879900.413520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879900.413614:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879900.669530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879901.437516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879901.437617:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879901.693484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879902.461496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879902.461612:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879902.717484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879903.485523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879903.485667:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879903.741523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879903.869538:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879903.869552:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879903.869559:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879903.869575:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4890 00000100:00080000:1.0:1788879903.869592:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879903.869604:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879903.869656:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511682048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879903.869716:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879903.869720:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879903.869722:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879903.869728:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4890 00000100:00080000:1.0:1788879903.869733:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879903.869738:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879903.869751:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879903.869768:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511682176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879903.869782:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511682048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879903.869785:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879903.869797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879903.869800:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879903.869805:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4890 00000100:00080000:1.0:1788879903.869808:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879903.869812:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879903.869819:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879903.869835:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511682304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879903.869868:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879903.869873:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879903.869876:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879903.869880:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4890 00000100:00080000:1.0:1788879903.869904:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879903.869909:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879903.869917:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879903.869936:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511682432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879903.869957:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879903.869962:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879903.869965:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879903.869971:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682176:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879903.869976:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4890 00000100:00080000:1.0:1788879903.869991:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879903.869998:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879903.870020:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879903.870035:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788879903.870063:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511682560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879903.870100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879903.870172:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879904.509532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879904.509628:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879904.765538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879905.533553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879905.533618:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879905.789514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879906.557517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879906.557675:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879906.813516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879906.941764:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879906.941817:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879906.941861:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879906.941925:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879906.941936:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879906.941945:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879906.941971:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879906.942083:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879903/real 1788879906] req@ffffa06188698380 x1875771511682560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879906.942120:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511682560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879906.942136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879906.942143:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879906.942148:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879906.942158:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879903/real 1788879906] req@ffffa0618869aa00 x1875771511682176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879906.942173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511682176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879906.942183:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879906.942187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879906.942191:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879906.942199:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879903/real 1788879906] req@ffffa06188699c00 x1875771511682304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879906.942214:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511682304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879906.942223:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879906.942227:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879906.942231:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879906.942238:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879903/real 1788879906] req@ffffa0618869b800 x1875771511682432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879906.942251:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511682432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879906.942259:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879906.942263:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879906.942267:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879906.942274:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879903/real 1788879906] req@ffffa0618869b100 x1875771511682048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879906.942287:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511682048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879906.942296:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879906.942299:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879906.942303:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511682048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879907.581482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879907.581552:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879907.837470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879908.605505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879908.605640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879908.861537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788879908.989601:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879908.989613:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788879908.989618:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879908.989632:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4895 00000100:00080000:3.0:1788879908.989646:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879908.989656:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788879908.989702:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511682688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879908.989736:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879908.989740:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788879908.989743:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879908.989748:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4895 00000100:00080000:3.0:1788879908.989753:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879908.989758:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879908.989773:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879908.989796:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511682816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879908.989798:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511682688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788879908.989847:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879908.989852:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788879908.989855:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879908.989861:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4895 00000100:00080000:3.0:1788879908.989867:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879908.989873:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879908.989881:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879908.989899:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511682944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879908.989913:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879908.989918:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788879908.989924:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879908.989929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788879908.989938:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4895 00000100:00080000:3.0:1788879908.989949:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879908.989959:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788879908.989970:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879908.990006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879908/real 1788879908] req@ffffa06188871c00 x1875771511682688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879908.990007:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511683072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788879908.990079:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879908.990088:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788879908.990095:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879908.990099:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511682688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879908.990107:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4895 00000100:00080000:3.0:1788879908.990121:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879908.990132:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879908.990134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788879908.990142:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879908.990147:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879908.990156:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511682688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788879908.990173:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511683200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879908.990200:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879908.990257:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879908.990285:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879908/real 1788879908] req@ffffa06188873800 x1875771511682816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879908.990311:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511682816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879908.990330:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879908.990337:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879908.990344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879908.990357:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879908.990446:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879908.990474:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879908/real 1788879908] req@ffffa06188871880 x1875771511682944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879908.990502:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511682944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879908.990519:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879908.990526:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879908.990532:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511682944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879908.990545:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879908.990588:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879908.990611:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879908/real 1788879908] req@ffffa06188870380 x1875771511683072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879908.990637:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511683072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879908.990654:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879908.990661:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879908.990668:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879908.990703:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879908.990746:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879908.990769:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879908/real 1788879908] req@ffffa06188870a80 x1875771511683200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879908.990794:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511683200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879908.990858:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879908.990866:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879908.990873:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879909.629570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879909.629665:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879909.885499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879910.653521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879910.653645:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879910.909499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879911.677544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879911.677689:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879911.933621:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879912.701520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879912.701714:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879912.957571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879913.725470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879913.725595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879913.981506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879914.109568:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879914.109584:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879914.109591:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879914.109608:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4900 00000100:00080000:2.0:1788879914.109625:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879914.109636:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879914.109710:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511683328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879914.109767:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879914.109795:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879914.109801:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879914.109810:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4900 00000100:00080000:2.0:1788879914.109819:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879914.109825:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879914.109842:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879914.109847:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511683328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879914.109869:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511683456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879914.109898:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879914.109905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879914.109910:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879914.109918:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4900 00000100:00080000:2.0:1788879914.109928:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879914.109938:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879914.109952:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788879914.109963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879914.109997:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511683584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879914.110015:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879914.110020:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879914.110022:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879914.110029:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4900 00000100:00080000:2.0:1788879914.110035:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879914.110037:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879914/real 1788879914] req@ffffa061888ae300 x1875771511683328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879914.110063:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879914.110073:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511683328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879914.110084:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879914.110091:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879914.110100:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879914.110108:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511683328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879914.110123:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511683712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879914.110148:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683456:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788879914.110155:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879914.110163:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879914.110168:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879914.110178:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4900 00000100:00080000:2.0:1788879914.110187:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879914.110199:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879914.110206:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879914.110219:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879914.110232:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879914/real 1788879914] req@ffffa061888adc00 x1875771511683456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879914.110250:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511683840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879914.110257:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511683456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879914.110346:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879914.110356:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879914.110365:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879914.110412:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879914.110440:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879914.110454:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879914/real 1788879914] req@ffffa061888ad500 x1875771511683584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879914.110469:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511683584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879914.110480:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879914.110484:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879914.110488:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879914.110506:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879914.110526:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879914.110539:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879914/real 1788879914] req@ffffa061888ac000 x1875771511683712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879914.110553:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511683712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879914.110563:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879914.110567:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879914.110572:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879914.110580:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879914.110598:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879914.110609:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879914/real 1788879914] req@ffffa061888adf80 x1875771511683840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879914.110623:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511683840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879914.110634:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879914.110637:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879914.110641:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511683840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879914.749476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879914.749605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879915.005518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879915.773522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879915.773704:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879916.029531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879916.797484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879916.797675:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879917.053539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879917.821534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879917.821678:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879918.077573:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879918.845477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879918.845716:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879919.101466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879919.229490:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879919.229499:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879919.229504:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879919.229535:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4905 00000100:00080000:2.0:1788879919.229555:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879919.229570:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879919.229620:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511683968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879919.229656:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879919.229661:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879919.229663:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879919.229668:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4905 00000100:00080000:2.0:1788879919.229672:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879919.229678:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879919.229708:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879919.229722:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511684096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879919.229752:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879919.229756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879919.229758:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879919.229761:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4905 00000100:00080000:2.0:1788879919.229765:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879919.229785:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511683968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879919.229787:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879919.229793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879919.229813:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511684224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879919.229820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879919.229823:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879919.229826:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879919.229829:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4905 00000100:00080000:2.0:1788879919.229831:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879919.229834:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879919.229838:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879919.229848:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511684352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879919.229855:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879919.229858:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879919.229860:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879919.229862:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4905 00000100:00080000:2.0:1788879919.229865:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879919.229869:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879919.229874:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879919.229881:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511684480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879919.229917:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879919.229987:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879919/real 1788879919] req@ffffa061888af480 x1875771511683968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879919.230003:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511683968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879919.230014:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879919.230019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879919.230072:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511683968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879919.230133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879919.230171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879919.230189:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879919/real 1788879919] req@ffffa061888aca80 x1875771511684096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879919.230204:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511684096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879919.230216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879919.230220:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879919.230224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879919.230254:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879919.230267:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879919.230274:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879919/real 1788879919] req@ffffa061888adf80 x1875771511684224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879919.230282:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511684224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879919.230287:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879919.230289:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879919.230291:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879919.230324:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879919.230345:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879919.230357:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879919/real 1788879919] req@ffffa061888ac000 x1875771511684352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879919.230370:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511684352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879919.230380:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879919.230383:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879919.230387:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879919.230393:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879919.230411:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879919.230454:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879919/real 1788879919] req@ffffa061888ad500 x1875771511684480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879919.230469:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511684480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879919.230479:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879919.230482:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879919.230486:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879919.869537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879919.869682:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879920.125495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879920.893478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879920.893591:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879921.149455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879921.917479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879921.917563:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879922.173514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879922.941506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879922.941639:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879923.197487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879923.965460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879923.965541:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879924.221485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879924.349597:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879924.349607:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879924.349612:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879924.349625:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4911 00000100:00080000:1.0:1788879924.349654:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879924.349663:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879924.349723:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511684608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879924.349759:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879924.349763:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879924.349765:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879924.349769:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4911 00000100:00080000:1.0:1788879924.349773:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879924.349777:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879924.349789:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879924.349807:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511684736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879924.349809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511684608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879924.349832:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879924.349836:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879924.349838:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879924.349841:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4911 00000100:00080000:1.0:1788879924.349844:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879924.349847:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879924.349854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879924.349867:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511684864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879924.349877:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879924.349881:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879924.349883:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879924.349886:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4911 00000100:00080000:1.0:1788879924.349898:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879924.349902:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879924.349907:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879924.349908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879924.349922:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511684992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879924.349933:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879924.349937:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879924.349939:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879924.349944:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4911 00000100:00080000:1.0:1788879924.349947:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879924.349961:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879924.349968:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879924.349980:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879924/real 1788879924] req@ffffa06188699f80 x1875771511684608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879924.349988:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511685120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879924.350010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511684608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879924.350023:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879924.350027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879924.350034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511684608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879924.350087:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879924.350119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879924.350135:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879924/real 1788879924] req@ffffa06188698000 x1875771511684736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879924.350150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511684736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879924.350161:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879924.350165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879924.350169:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879924.350177:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879924.350195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879924.350206:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879924/real 1788879924] req@ffffa0618869a680 x1875771511684864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879924.350237:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511684864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879924.350252:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879924.350257:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879924.350262:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879924.350271:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879924.350300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879924.350318:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879924/real 1788879924] req@ffffa0618869bb80 x1875771511684992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879924.350338:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511684992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879924.350353:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879924.350357:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879924.350361:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511684992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879924.350368:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879924.350401:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879924.350413:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879924/real 1788879924] req@ffffa0618869ad80 x1875771511685120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879924.350471:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511685120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879924.350480:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879924.350484:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879924.350488:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879924.989476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879924.989561:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879925.245479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879926.013452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879926.013522:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879926.269452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879927.037456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879927.037560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879927.293479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879928.061491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879928.061597:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879928.317500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879929.085512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879929.085596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879929.341475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879929.469478:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879929.469488:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879929.469492:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879929.469502:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4916 00000100:00080000:1.0:1788879929.469512:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879929.469519:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879929.469553:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511685248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879929.469576:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879929.469578:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879929.469580:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879929.469583:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4916 00000100:00080000:1.0:1788879929.469585:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879929.469588:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879929.469596:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879929.469605:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511685376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879929.469614:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879929.469617:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879929.469620:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879929.469621:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511685248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879929.469624:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4916 00000100:00080000:1.0:1788879929.469629:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879929.469633:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879929.469636:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879929.469644:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511685504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879929.469649:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879929.469652:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879929.469654:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879929.469656:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4916 00000100:00080000:1.0:1788879929.469659:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879929.469677:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879929.469681:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879929.469691:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511685632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879929.469698:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879929.469701:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879929.469702:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879929.469705:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4916 00000100:00080000:1.0:1788879929.469708:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879929.469713:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788879929.469716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788879929.469718:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879929.469727:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511685760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879929.469762:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879929/real 1788879929] req@ffffa0618869ad80 x1875771511685248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879929.469783:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511685248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879929.469795:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879929.469800:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879929.469804:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511685248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879929.469830:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879929.469855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879929.469869:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879929/real 1788879929] req@ffffa0618869bb80 x1875771511685376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879929.469882:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511685376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879929.469890:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879929.469893:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879929.469898:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879929.469904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879929.469936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879929.469954:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879929/real 1788879929] req@ffffa0618869a680 x1875771511685504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879929.469967:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511685504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879929.469976:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879929.469979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879929.469983:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879929.469989:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879929.470012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879929.470024:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879929/real 1788879929] req@ffffa06188698000 x1875771511685632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879929.470036:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511685632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879929.470045:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879929.470048:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879929.470051:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879929.470057:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879929.470079:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879929.470092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879929/real 1788879929] req@ffffa06188699f80 x1875771511685760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879929.470104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511685760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879929.470113:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879929.470116:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879929.470119:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511685760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879930.109464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879930.109546:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879930.365486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879931.133496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879931.133563:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879931.389446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879932.157459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879932.157512:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879932.413456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879933.181478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879933.181531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879933.437503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879934.205467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879934.205521:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879934.461443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879934.589515:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879934.589524:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879934.589528:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879934.589539:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4921 00000100:00080000:1.0:1788879934.589550:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879934.589567:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879934.589607:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511685888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879934.589645:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879934.589649:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879934.589651:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879934.589667:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4921 00000100:00080000:1.0:1788879934.589672:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879934.589676:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879934.589685:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511685888:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879934.589686:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879934.589699:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511686016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879934.589709:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879934.589712:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879934.589714:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879934.589717:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4921 00000100:00080000:1.0:1788879934.589732:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879934.589735:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879934.589740:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879934.589759:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511686144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879934.589764:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879934.589773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879934.589776:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879934.589777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879934.589781:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4921 00000100:00080000:1.0:1788879934.589784:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879934.589797:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879934.589802:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879934.589810:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879934/real 1788879934] req@ffffa0618869b100 x1875771511685888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879934.589811:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511686272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879934.589822:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879934.589826:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879934.589827:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879934.589830:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4921 00000100:00100000:0.0:1788879934.589831:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511685888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879934.589834:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879934.589838:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879934.589842:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788879934.589842:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879934.589847:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879934.589862:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511685888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879934.589862:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511686400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879934.589885:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879934.589909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879934.589922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879934/real 1788879934] req@ffffa0618869b800 x1875771511686016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879934.589931:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511686016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879934.589938:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879934.589940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879934.589950:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879934.589955:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879934.589970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879934.589977:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879934/real 1788879934] req@ffffa06188699c00 x1875771511686144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879934.589985:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511686144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879934.589990:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879934.589991:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879934.589993:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879934.589997:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879934.590008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879934.590013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879934/real 1788879934] req@ffffa0618869aa00 x1875771511686272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879934.590020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511686272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879934.590026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879934.590028:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879934.590030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879934.590033:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879934.590042:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879934.590048:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879934/real 1788879934] req@ffffa06188698380 x1875771511686400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879934.590054:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511686400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879934.590059:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879934.590061:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879934.590063:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879935.229457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879935.229509:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879935.485601:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879936.253446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879936.253554:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879936.509517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879937.277505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879937.277620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879937.533535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879938.301484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879938.301621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879938.557481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879939.325539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879939.325682:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879939.581471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879939.709524:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879939.709532:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879939.709536:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879939.709545:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4926 00000100:00080000:2.0:1788879939.709554:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879939.709559:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879939.709621:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511686528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879939.709641:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879939.709644:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879939.709646:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879939.709649:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4926 00000100:00080000:2.0:1788879939.709652:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879939.709654:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879939.709663:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879939.709672:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511686656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879939.709680:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879939.709683:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879939.709684:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879939.709690:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4926 00000100:00100000:0.0:1788879939.709692:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511686528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879939.709693:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879939.709698:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879939.709705:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879939.709713:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511686784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879939.709720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879939.709723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879939.709724:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879939.709726:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4926 00000100:00080000:2.0:1788879939.709728:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879939.709730:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879939.709734:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879939.709740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511686912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879939.709746:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879939.709748:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879939.709749:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879939.709752:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4926 00000100:00080000:2.0:1788879939.709754:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879939.709756:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879939.709759:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879939.709766:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511687040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879939.709774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879939.709815:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879939/real 1788879939] req@ffffa061888adc00 x1875771511686528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879939.709837:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511686528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879939.709849:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879939.709853:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879939.709858:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511686528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879939.709881:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879939.709906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879939.709919:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879939/real 1788879939] req@ffffa061888ae300 x1875771511686656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879939.709933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511686656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879939.709943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879939.709947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879939.709950:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879939.709957:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879939.709978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879939.709990:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879939/real 1788879939] req@ffffa061888aea00 x1875771511686784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879939.710004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511686784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879939.710022:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879939.710026:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879939.710030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879939.710037:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879939.710059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879939.710071:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879939/real 1788879939] req@ffffa061888afb80 x1875771511686912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879939.710084:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511686912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879939.710093:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879939.710096:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879939.710099:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511686912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879939.710106:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879939.710127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879939.710138:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879939/real 1788879939] req@ffffa061888af100 x1875771511687040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879939.710151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511687040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879939.710160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879939.710163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879939.710167:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879940.349455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879940.349500:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879940.605464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879941.373473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879941.373537:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879941.629511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879942.397479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879942.397545:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879942.653541:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879943.421513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879943.421638:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879943.677490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:2.0:1788879944.445639:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788879944.445711:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788879944.701508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879944.829589:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879944.829600:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879944.829606:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879944.829622:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4931 00000100:00080000:1.0:1788879944.829639:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879944.829651:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879944.829704:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511687168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879944.829735:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879944.829739:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879944.829741:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879944.829746:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4931 00000100:00080000:1.0:1788879944.829751:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879944.829758:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879944.829769:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879944.829785:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511687168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879944.829788:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511687296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879944.829807:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879944.829810:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879944.829812:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879944.829816:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4931 00000100:00080000:1.0:1788879944.829819:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879944.829823:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879944.829827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879944.829846:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511687424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879944.829862:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879944.829868:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879944.829871:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879944.829878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879944.829884:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4931 00000100:00080000:1.0:1788879944.829895:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879944.829903:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879944.829913:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879944.829928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879944/real 1788879944] req@ffffa06188698380 x1875771511687168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879944.829932:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511687552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879944.829979:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879944.829983:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879944.829986:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879944.829992:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511687168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879944.829996:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4931 00000100:00080000:1.0:1788879944.830005:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879944.830014:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879944.830018:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788879944.830020:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879944.830027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879944.830034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511687168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879944.830039:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511687680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879944.830069:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879944.830095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879944.830108:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879944/real 1788879944] req@ffffa0618869aa00 x1875771511687296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879944.830120:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511687296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879944.830129:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879944.830133:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879944.830136:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879944.830142:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879944.830157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879944.830166:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879944/real 1788879944] req@ffffa06188699c00 x1875771511687424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879944.830179:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511687424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879944.830187:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879944.830190:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879944.830193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879944.830198:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879944.830213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879944.830222:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879944/real 1788879944] req@ffffa0618869b800 x1875771511687552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879944.830234:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511687552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879944.830242:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879944.830244:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879944.830247:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879944.830252:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879944.830266:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879944.830275:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879944/real 1788879944] req@ffffa0618869b100 x1875771511687680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879944.830286:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511687680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879944.830301:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879944.830304:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879944.830307:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879945.469537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879945.469645:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879945.725490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879946.493531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879946.493693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879946.749565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879947.517529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879947.517670:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879947.773509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879948.541512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879948.541690:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879948.797483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879949.565507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879949.565679:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879949.821514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879949.949600:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879949.949614:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879949.949620:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879949.949633:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4936 00000100:00080000:1.0:1788879949.949658:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879949.949666:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879949.949712:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511687808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879949.949756:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879949.949760:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879949.949762:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879949.949767:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4936 00000100:00080000:1.0:1788879949.949771:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879949.949776:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879949.949789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879949.949833:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511687808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879949.949839:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511687936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879949.949859:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879949.949864:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879949.949866:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879949.949871:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4936 00000100:00080000:1.0:1788879949.949876:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879949.949881:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879949.949888:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879949.949904:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511688064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879949.949916:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879949.949921:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879949.949923:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879949.949929:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4936 00000800:00000100:0.0:1788879949.949938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879949.949941:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879949.949950:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879949.949958:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879949.949974:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511688192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879949.949997:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879949.950005:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788879949.950006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879949/real 1788879949] req@ffffa06188699f80 x1875771511687808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879949.950009:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879949.950026:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4936 00000100:00080000:1.0:1788879949.950039:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879949.950046:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879949.950052:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879949.950057:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511687808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879949.950071:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511688320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879949.950079:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879949.950090:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879949.950096:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511687808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879949.950140:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879949.950169:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879949.950184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879949/real 1788879949] req@ffffa06188698000 x1875771511687936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879949.950206:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511687936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879949.950224:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879949.950231:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879949.950237:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511687936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879949.950265:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879949.950297:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879949.950317:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879949/real 1788879949] req@ffffa0618869a680 x1875771511688064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879949.950336:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511688064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879949.950348:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879949.950354:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879949.950359:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879949.950386:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879949.950424:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879949.950490:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879949/real 1788879949] req@ffffa0618869bb80 x1875771511688192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879949.950516:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511688192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879949.950584:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879949.950590:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879949.950597:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879949.950608:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879949.950652:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879949.950672:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879949/real 1788879949] req@ffffa0618869ad80 x1875771511688320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879949.950703:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511688320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879949.950720:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879949.950726:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879949.950732:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879950.589582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879950.589684:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879950.845558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879951.613502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879951.613663:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879951.869554:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879952.637517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879952.637610:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879952.893585:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879953.661526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879953.661650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879953.917553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879954.685465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879954.685569:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879954.941566:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879955.069568:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879955.069582:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879955.069588:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879955.069603:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4941 00000100:00080000:2.0:1788879955.069618:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879955.069628:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879955.069677:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511688448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879955.069715:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879955.069720:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879955.069722:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879955.069728:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4941 00000100:00080000:2.0:1788879955.069734:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879955.069740:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879955.069754:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879955.069774:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511688448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788879955.069776:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511688576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879955.069802:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879955.069806:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879955.069809:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879955.069815:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4941 00000100:00080000:2.0:1788879955.069820:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879955.069824:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879955.069830:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879955.069846:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511688704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879955.069860:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879955.069865:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879955.069867:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879955.069871:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4941 00000100:00080000:2.0:1788879955.069876:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879955.069883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879955.069888:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879955.069897:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879955.069921:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511688832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879955.069938:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879955/real 1788879955] req@ffffa061888aed80 x1875771511688448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879955.069970:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511688448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879955.069996:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879955.069999:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879955.070011:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788879955.070012:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788879955.070023:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511688448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879955.070023:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879955.070044:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4941 00000100:00080000:2.0:1788879955.070054:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879955.070061:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879955.070069:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688576:192.168.203.111@tcp:38:kworker.0 00000008:00100000:2.0:1788879955.070075:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879955.070107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788879955.070112:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511688960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879955.070129:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879955/real 1788879955] req@ffffa061888ac700 x1875771511688576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879955.070156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511688576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879955.070168:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879955.070173:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879955.070180:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879955.070188:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879955.070208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879955.070220:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879955/real 1788879955] req@ffffa061888ac380 x1875771511688704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879955.070233:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511688704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879955.070243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879955.070246:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879955.070250:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879955.070257:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879955.070275:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879955.070287:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879955/real 1788879955] req@ffffa061888ad880 x1875771511688832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879955.070301:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511688832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879955.070310:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879955.070314:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879955.070317:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879955.070333:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879955.070351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879955.070363:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879955/real 1788879955] req@ffffa061888ae680 x1875771511688960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879955.070376:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511688960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879955.070417:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879955.070420:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879955.070425:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511688960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879955.709463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879955.709530:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879955.965473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879956.733563:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879956.733629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879956.989589:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879957.757625:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879957.757740:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879958.013544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879958.781529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879958.781573:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879959.037471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879959.805496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879959.805555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879960.061461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879960.189508:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879960.189518:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879960.189523:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879960.189533:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4946 00000100:00080000:1.0:1788879960.189544:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879960.189552:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879960.189593:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511689088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879960.189620:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879960.189627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879960.189630:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879960.189636:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4946 00000100:00080000:1.0:1788879960.189640:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879960.189645:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879960.189651:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511689088:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788879960.189655:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879960.189673:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511689216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879960.189685:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879960.189688:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879960.189690:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879960.189694:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4946 00000100:00080000:1.0:1788879960.189706:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788879960.189712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879960.189713:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879960.189721:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879960.189736:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511689344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879960.189746:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879960.189751:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1788879960.189752:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879960/real 1788879960] req@ffffa0618869ad80 x1875771511689088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879960.189754:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879960.189763:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4946 00000100:00080000:1.0:1788879960.189771:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879960.189777:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879960.189784:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879960.189785:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511689088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879960.189797:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788879960.189801:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511689472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879960.189802:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879960.189808:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511689088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879960.189822:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879960.189826:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879960.189829:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879960.189833:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689216:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788879960.189835:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4946 00000100:00080000:1.0:1788879960.189842:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879960.189846:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879960.189853:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879960.189857:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879960.189874:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879960/real 1788879960] req@ffffa0618869bb80 x1875771511689216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879960.189875:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511689600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879960.189896:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511689216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879960.189905:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879960.189908:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879960.189910:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879960.189914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879960.189925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879960.189933:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879960/real 1788879960] req@ffffa0618869a680 x1875771511689344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879960.189941:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511689344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879960.189951:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879960.189953:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879960.189955:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879960.189958:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879960.189968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879960.189974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879960/real 1788879960] req@ffffa06188698000 x1875771511689472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879960.189981:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511689472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879960.189986:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879960.189988:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879960.189989:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879960.189998:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879960.190008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879960.190014:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879960/real 1788879960] req@ffffa06188699f80 x1875771511689600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879960.190021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511689600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879960.190026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879960.190028:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879960.190030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879960.829460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879960.829552:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879961.085542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879961.853490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879961.853647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879962.109499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879962.877557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879962.877659:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879963.133519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879963.901515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879963.901581:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879964.157542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879964.925552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879964.925598:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879965.181448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879965.309502:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879965.309513:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879965.309518:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879965.309533:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4952 00000100:00080000:1.0:1788879965.309550:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879965.309562:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879965.309611:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511689728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879965.309655:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879965.309660:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879965.309663:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879965.309679:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4952 00000100:00080000:1.0:1788879965.309684:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879965.309689:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879965.309702:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879965.309719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511689856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879965.309733:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879965.309740:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788879965.309742:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511689728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879965.309743:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879965.309753:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4952 00000100:00080000:1.0:1788879965.309760:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879965.309767:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879965.309774:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879965.309788:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511689984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879965.309800:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879965.309805:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879965.309807:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879965.309812:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4952 00000100:00080000:1.0:1788879965.309840:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879965.309846:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879965.309853:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879965.309859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879965.309874:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511690112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879965.309889:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879965.309894:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879965.309897:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879965.309903:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4952 00000100:00080000:1.0:1788879965.309907:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879965.309912:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879965.309920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879965/real 1788879965] req@ffffa0618869b100 x1875771511689728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879965.309923:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879965.309945:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511690240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879965.309962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511689728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879965.309982:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879965.309988:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879965.309994:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511689728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879965.310033:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879965.310061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879965.310077:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879965/real 1788879965] req@ffffa0618869b800 x1875771511689856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879965.310092:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511689856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879965.310112:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879965.310117:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879965.310122:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879965.310130:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879965.310149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879965.310160:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879965/real 1788879965] req@ffffa06188699c00 x1875771511689984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879965.310174:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511689984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879965.310184:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879965.310187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879965.310191:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511689984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879965.310198:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879965.310217:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879965.310229:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879965/real 1788879965] req@ffffa0618869aa00 x1875771511690112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879965.310243:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511690112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879965.310252:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879965.310256:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879965.310259:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879965.310266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879965.310285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879965.310300:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879965/real 1788879965] req@ffffa06188698380 x1875771511690240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879965.310314:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511690240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879965.310323:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879965.310327:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879965.310330:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879965.949556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879965.949722:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879966.205560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879966.973549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879966.973746:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879967.229493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879967.997497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879967.997579:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879968.253548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879969.021566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879969.021658:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879969.277508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879970.045528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879970.045621:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879970.301461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879970.429466:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879970.429474:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879970.429478:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879970.429487:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4957 00000100:00080000:1.0:1788879970.429497:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879970.429504:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879970.429539:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511690368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879970.429562:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879970.429565:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879970.429566:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879970.429569:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4957 00000100:00080000:1.0:1788879970.429572:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879970.429580:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879970.429589:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879970.429595:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511690368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879970.429601:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511690496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879970.429610:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879970.429613:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879970.429614:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879970.429617:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4957 00000100:00080000:1.0:1788879970.429619:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879970.429621:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879970.429625:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879970.429633:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511690624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879970.429640:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879970.429643:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879970.429644:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879970.429647:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4957 00000100:00080000:1.0:1788879970.429657:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879970.429660:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879970.429663:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879970.429671:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511690752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879970.429677:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879970.429680:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879970.429681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879970.429684:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4957 00000100:00080000:1.0:1788879970.429687:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879970.429689:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879970.429692:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879970.429699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511690880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879970.429704:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879970.429733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879970.429747:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879970.429768:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879971.069468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879971.069535:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879971.325453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879972.093478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879972.093543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879972.349477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788879973.117451:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879973.117525:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879973.373437:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788879973.501662:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788879973.501696:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879973.501710:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879973.501741:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879973.501750:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879973.501761:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788879973.501767:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788879973.501780:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879970/real 1788879973] req@ffffa0618869aa00 x1875771511690496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879973.501808:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511690496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879973.501823:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879973.501828:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879973.501833:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879973.501840:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879970/real 1788879973] req@ffffa06188699c00 x1875771511690624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879973.501853:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511690624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879973.501861:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879973.501865:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879973.501869:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879973.501875:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879970/real 1788879973] req@ffffa0618869b800 x1875771511690752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879973.501888:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511690752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879973.501896:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879973.501900:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879973.501903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879973.501909:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879970/real 1788879973] req@ffffa06188698380 x1875771511690368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879973.501922:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511690368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879973.501930:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879973.501934:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879973.501937:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511690368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879973.501963:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879970/real 1788879973] req@ffffa0618869b100 x1875771511690880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879973.501975:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511690880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879973.501982:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879973.501986:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879973.501989:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511690880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879974.141480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879974.141543:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879974.397463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879975.165483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879975.165484:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879975.421454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879975.549496:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879975.549509:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879975.549514:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879975.549526:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4962 00000100:00080000:1.0:1788879975.549538:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879975.549547:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879975.549584:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511691008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879975.549610:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879975.549615:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879975.549616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879975.549621:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4962 00000100:00080000:1.0:1788879975.549624:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879975.549629:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879975.549640:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879975.549653:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511691008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879975.549658:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511691136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879975.549673:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879975.549676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879975.549678:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879975.549682:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4962 00000100:00080000:1.0:1788879975.549686:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879975.549690:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879975.549697:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879975.549709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511691264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879975.549720:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879975.549724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879975.549726:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788879975.549730:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879975.549735:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4962 00000100:00080000:1.0:1788879975.549740:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879975.549751:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879975.549758:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879975.549773:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511691392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879975.549776:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879975/real 1788879975] req@ffffa06188699f80 x1875771511691008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879975.549796:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879975.549801:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879975.549804:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788879975.549811:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511691008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879975.549812:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4962 00000100:00080000:1.0:1788879975.549820:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879975.549829:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788879975.549831:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1788879975.549838:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788879975.549839:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879975.549847:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511691008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879975.549858:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511691520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879975.549879:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879975.549906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879975.549920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879975/real 1788879975] req@ffffa06188698000 x1875771511691136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879975.549934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511691136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879975.549943:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879975.549947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879975.549951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879975.549958:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879975.549976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879975.549987:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879975/real 1788879975] req@ffffa0618869a680 x1875771511691264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879975.549999:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511691264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879975.550009:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879975.550013:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879975.550017:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879975.550024:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879975.550044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879975.550055:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879975/real 1788879975] req@ffffa0618869bb80 x1875771511691392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879975.550076:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511691392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879975.550084:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879975.550088:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879975.550091:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879975.550097:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879975.550115:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879975.550125:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879975/real 1788879975] req@ffffa0618869ad80 x1875771511691520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879975.550137:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511691520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879975.550147:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879975.550150:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879975.550153:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879976.189471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879976.189579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879976.445462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879977.213487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879977.213548:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879977.469444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879978.237471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879978.237548:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879978.493515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879979.261480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879979.261552:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879979.517462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879980.285509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879980.285640:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879980.541535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788879980.669549:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879980.669560:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788879980.669566:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879980.669580:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4967 00000100:00080000:2.0:1788879980.669596:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879980.669607:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788879980.669656:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511691648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879980.669692:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879980.669697:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788879980.669700:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879980.669706:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4967 00000100:00080000:2.0:1788879980.669710:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879980.669716:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879980.669730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879980.669748:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511691776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879980.669765:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879980.669768:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511691648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788879980.669770:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788879980.669777:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879980.669791:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4967 00000100:00080000:2.0:1788879980.669796:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879980.669801:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879980.669807:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879980.669825:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511691904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879980.669837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879980.669842:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788879980.669844:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879980.669849:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4967 00000100:00080000:2.0:1788879980.669854:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879980.669858:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879980.669863:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788879980.669897:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511692032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879980.669899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788879980.669917:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879980.669923:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879980.669925:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879980.669931:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4967 00000100:00080000:2.0:1788879980.669935:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788879980.669940:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788879980.669948:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879980.669962:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879980/real 1788879980] req@ffffa061888ad180 x1875771511691648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879980.669966:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511692160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879980.670027:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511691648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879980.670053:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879980.670058:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879980.670064:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511691648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879980.670094:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879980.670124:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879980.670139:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879980/real 1788879980] req@ffffa061888ace00 x1875771511691776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879980.670155:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511691776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879980.670165:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879980.670169:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879980.670173:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879980.670180:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879980.670200:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879980.670211:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879980/real 1788879980] req@ffffa061888ae680 x1875771511691904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879980.670224:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511691904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879980.670234:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879980.670238:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879980.670241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511691904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879980.670249:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879980.670268:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879980.670278:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879980/real 1788879980] req@ffffa061888ad880 x1875771511692032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879980.670292:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511692032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879980.670332:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879980.670336:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879980.670340:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879980.670347:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879980.670369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879980.670412:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879980/real 1788879980] req@ffffa061888ac380 x1875771511692160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879980.670428:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511692160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879980.670438:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879980.670442:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879980.670446:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879981.309474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879981.309557:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879981.565435:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879982.333457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879982.333518:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879982.589453:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879983.357448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879983.357506:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879983.613475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879984.381446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879984.381507:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879984.637485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879985.405555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879985.405720:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879985.661524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879985.789549:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879985.789560:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879985.789567:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879985.789582:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4972 00000100:00080000:1.0:1788879985.789597:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879985.789607:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879985.789658:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511692288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879985.789697:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879985.789702:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879985.789704:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879985.789710:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4972 00000100:00080000:1.0:1788879985.789715:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879985.789721:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879985.789735:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879985.789753:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511692288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879985.789757:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511692416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879985.789787:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879985.789793:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879985.789798:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879985.789806:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4972 00000100:00080000:1.0:1788879985.789813:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879985.789820:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879985.789831:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879985.789857:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511692544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788879985.789870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879985.789889:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879985.789895:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879985.789898:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879985.789905:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4972 00000100:00080000:1.0:1788879985.789913:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879985.789919:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788879985.789928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879985/real 1788879985] req@ffffa0618869ad80 x1875771511692288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879985.789930:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879985.789961:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511692672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879985.789969:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511692288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879985.789985:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879985.789999:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879985.790002:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788879985.790004:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879985.790012:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4972 00000100:00080000:0.0:1788879985.790020:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788879985.790022:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879985.790034:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511692288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788879985.790034:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879985.790047:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879985.790064:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511692800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879985.790080:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879985.790113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879985.790130:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879985/real 1788879985] req@ffffa0618869bb80 x1875771511692416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879985.790146:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511692416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879985.790157:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879985.790161:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879985.790165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879985.790172:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879985.790192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879985.790203:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879985/real 1788879985] req@ffffa0618869a680 x1875771511692544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879985.790217:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511692544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879985.790235:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879985.790239:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879985.790244:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879985.790251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879985.790298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879985.790314:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879985/real 1788879985] req@ffffa06188698000 x1875771511692672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879985.790329:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511692672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879985.790340:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879985.790344:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879985.790348:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879985.790368:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879985.790425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879985.790440:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879985/real 1788879985] req@ffffa06188699f80 x1875771511692800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879985.790455:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511692800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879985.790465:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879985.790469:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879985.790474:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511692800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879986.429547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879986.429636:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879986.685478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879987.453495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879987.453613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879987.709494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879988.477509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879988.477600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879988.733499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879989.501511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879989.501616:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879989.757488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879990.525552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879990.525673:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879990.781489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879990.909486:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879990.909496:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879990.909501:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879990.909515:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4977 00000100:00080000:1.0:1788879990.909533:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879990.909544:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879990.909596:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511692928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879990.909625:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879990.909630:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879990.909635:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879990.909641:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4977 00000100:00080000:1.0:1788879990.909646:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879990.909653:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879990.909666:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879990.909671:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511692928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879990.909686:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511693056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879990.909702:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879990.909707:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879990.909709:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879990.909714:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4977 00000100:00080000:1.0:1788879990.909719:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879990.909723:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879990.909729:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879990.909744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511693184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879990.909757:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879990.909761:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879990.909764:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879990.909773:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4977 00000800:00000100:0.0:1788879990.909775:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788879990.909779:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879990.909787:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879990.909794:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879990.909810:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511693312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879990.909822:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879990.909830:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788879990.909833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879990/real 1788879990] req@ffffa0618869b100 x1875771511692928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879990.909835:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879990.909849:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4977 00000100:00080000:1.0:1788879990.909859:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879990.909869:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879990.909876:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788879990.909887:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511692928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879990.909897:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511693440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788879990.909932:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879990.909943:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879990.909951:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511692928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879990.909982:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879990.910010:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879990.910027:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879990/real 1788879990] req@ffffa06188698380 x1875771511693056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879990.910043:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511693056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879990.910053:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879990.910057:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879990.910061:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879990.910068:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879990.910088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879990.910099:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879990/real 1788879990] req@ffffa0618869b800 x1875771511693184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879990.910113:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511693184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879990.910123:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879990.910126:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879990.910130:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879990.910136:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879990.910155:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879990.910166:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879990/real 1788879990] req@ffffa06188699c00 x1875771511693312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879990.910180:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511693312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879990.910190:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879990.910193:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879990.910197:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879990.910203:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879990.910221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879990.910264:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879990/real 1788879990] req@ffffa0618869aa00 x1875771511693440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879990.910288:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511693440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879990.910298:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879990.910301:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879990.910305:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879991.549478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879991.549686:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879991.805502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879992.573522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879992.573612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879992.829537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879993.597564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879993.597608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879993.853571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879994.621566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879994.621636:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879994.877536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879995.645535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879995.645654:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879995.901507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788879996.029525:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879996.029535:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788879996.029542:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879996.029560:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4982 00000100:00080000:1.0:1788879996.029578:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879996.029592:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788879996.029652:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511693568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879996.029694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879996.029701:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788879996.029704:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879996.029712:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4982 00000100:00080000:1.0:1788879996.029718:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879996.029724:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879996.029739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788879996.029759:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511693568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788879996.029760:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511693696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879996.029787:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879996.029791:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788879996.029794:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879996.029800:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4982 00000100:00080000:1.0:1788879996.029804:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879996.029809:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879996.029814:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879996.029832:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511693824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879996.029843:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879996.029848:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879996.029852:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879996.029858:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4982 00000100:00080000:1.0:1788879996.029863:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879996.029867:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879996.029874:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788879996.029875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788879996.029897:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511693952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879996.029910:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879996.029914:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879996.029917:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879996.029922:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4982 00000100:00080000:1.0:1788879996.029927:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788879996.029937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879996/real 1788879996] req@ffffa0618869aa00 x1875771511693568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879996.029938:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788879996.029953:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788879996.029978:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511694080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788879996.029984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511693568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879996.030023:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879996.030029:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879996.030035:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511693568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788879996.030072:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879996.030101:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879996.030116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879996/real 1788879996] req@ffffa06188699c00 x1875771511693696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879996.030131:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511693696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879996.030143:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879996.030148:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879996.030152:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879996.030159:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788879996.030178:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879996.030234:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879996/real 1788879996] req@ffffa0618869b800 x1875771511693824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879996.030249:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511693824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879996.030261:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879996.030264:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879996.030268:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788879996.030276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879996.030302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879996.030317:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879996/real 1788879996] req@ffffa06188698380 x1875771511693952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879996.030341:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511693952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879996.030357:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879996.030364:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879996.030372:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511693952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788879996.030423:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788879996.030457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788879996.030473:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879996/real 1788879996] req@ffffa0618869b100 x1875771511694080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879996.030489:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511694080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879996.030500:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879996.030504:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788879996.030508:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788879996.669511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879996.669625:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879996.925498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879997.693544:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879997.693637:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879997.949525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879998.717504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788879998.717649:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879998.973509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788879999.741512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788879999.741587:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788879999.997484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880000.765502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880000.765649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880001.021515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880001.149553:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880001.149567:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880001.149573:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880001.149589:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4987 00000100:00080000:2.0:1788880001.149605:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880001.149616:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880001.149663:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511694208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880001.149699:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880001.149704:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880001.149706:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880001.149711:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4987 00000100:00080000:2.0:1788880001.149716:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880001.149722:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880001.149735:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880001.149763:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511694208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880001.149763:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511694336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880001.149792:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880001.149797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880001.149800:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880001.149805:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4987 00000100:00080000:2.0:1788880001.149810:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880001.149815:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880001.149823:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880001.149839:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511694464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880001.149852:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880001.149857:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880001.149861:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880001.149870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880001.149879:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4987 00000100:00080000:2.0:1788880001.149891:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880001.149902:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880001.149914:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880001.149927:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880001/real 1788880001] req@ffffa061888ac700 x1875771511694208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880001.149944:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511694592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880001.149961:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511694208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880001.149983:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880001.149990:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880001.149991:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880001.150002:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880001.150005:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880001.150019:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511694208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880001.150024:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4987 00000100:00080000:2.0:1788880001.150039:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880001.150049:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880001.150059:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694336:192.168.203.111@tcp:38:kworker.0 00000008:00100000:2.0:1788880001.150062:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880001.150104:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511694720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880001.150115:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880001.150183:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880001/real 1788880001] req@ffffa061888aed80 x1875771511694336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880001.150205:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511694336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880001.150217:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880001.150221:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880001.150226:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880001.150234:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880001.150262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880001.150276:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880001/real 1788880001] req@ffffa061888af100 x1875771511694464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880001.150290:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511694464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880001.150302:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880001.150308:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880001.150314:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880001.150326:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880001.150364:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880001.150426:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880001/real 1788880001] req@ffffa061888afb80 x1875771511694592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880001.150455:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511694592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880001.150466:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880001.150470:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880001.150474:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880001.150499:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880001.150527:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880001.150541:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880001/real 1788880001] req@ffffa061888aea00 x1875771511694720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880001.150555:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511694720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880001.150565:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880001.150569:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880001.150573:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880001.789503:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880001.789665:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880002.045487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880002.813486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880002.813642:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880003.069468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880003.837469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880003.837555:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880004.093455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880004.861459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880004.861528:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880005.117526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880005.885498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880005.885644:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880006.141514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880006.269548:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880006.269562:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880006.269567:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880006.269582:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4992 00000100:00080000:1.0:1788880006.269597:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880006.269609:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880006.269658:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511694848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880006.269700:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880006.269706:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880006.269710:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880006.269717:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4992 00000100:00080000:1.0:1788880006.269724:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880006.269729:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880006.269742:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880006.269757:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511694848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880006.269767:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511694976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880006.269793:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880006.269798:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880006.269801:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880006.269806:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4992 00000100:00080000:1.0:1788880006.269812:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880006.269817:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880006.269824:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880006.269838:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511695104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880006.269850:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788880006.269861:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880006.269871:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880006.269875:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880006.269883:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4992 00000100:00080000:1.0:1788880006.269891:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880006.269897:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880006.269903:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880006.269919:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511695232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880006.269922:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880006/real 1788880006] req@ffffa06188699f80 x1875771511694848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880006.269983:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880006.269989:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880006.269992:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880006.270002:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511694848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880006.270009:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4992 00000100:00080000:1.0:1788880006.270023:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880006.270034:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788880006.270036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880006.270058:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:1.0:1788880006.270060:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880006.270077:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511694848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880006.270085:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511695360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880006.270120:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880006.270194:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880006.270210:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880006/real 1788880006] req@ffffa06188698000 x1875771511694976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880006.270221:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511694976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880006.270229:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880006.270234:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880006.270238:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511694976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880006.270245:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880006.270275:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880006.270283:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880006/real 1788880006] req@ffffa0618869a680 x1875771511695104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880006.270294:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511695104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880006.270300:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880006.270302:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880006.270304:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880006.270310:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880006.270328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880006.270336:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880006/real 1788880006] req@ffffa0618869bb80 x1875771511695232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880006.270344:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511695232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880006.270350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880006.270352:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880006.270355:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880006.270358:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880006.270372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880006.270378:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880006/real 1788880006] req@ffffa0618869ad80 x1875771511695360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880006.270385:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511695360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880006.270390:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880006.270392:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880006.270394:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880006.909454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880006.909545:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880007.165443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880007.933469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880007.933542:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880008.189436:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880008.957495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880008.957697:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880009.213451:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880009.981519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880009.981615:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880010.237498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880011.005516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880011.005639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880011.261486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880011.389587:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880011.389598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880011.389603:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880011.389618:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 4998 00000100:00080000:2.0:1788880011.389632:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880011.389641:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880011.389687:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511695488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880011.389726:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880011.389730:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880011.389733:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880011.389739:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4998 00000100:00080000:2.0:1788880011.389743:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880011.389747:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880011.389760:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880011.389778:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511695616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880011.389795:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511695488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880011.389798:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880011.389814:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880011.389820:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880011.389826:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4998 00000100:00080000:2.0:1788880011.389830:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880011.389835:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880011.389841:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880011.389859:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511695744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880011.389879:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880011.389884:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880011.389887:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880011.389892:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4998 00000100:00080000:2.0:1788880011.389896:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880011.389900:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880011.389908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788880011.389910:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880011.389931:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511695872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880011.389944:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880011.389950:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880011.389953:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880011.389963:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 4998 00000100:00100000:0.0:1788880011.389965:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880011/real 1788880011] req@ffffa061888ae300 x1875771511695488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880011.389968:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880011.389980:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880011.389993:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880011.390016:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511695488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880011.390017:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511696000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880011.390063:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880011.390073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880011.390080:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511695488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880011.390153:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880011.390189:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880011.390207:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880011/real 1788880011] req@ffffa061888adc00 x1875771511695616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880011.390223:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511695616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880011.390235:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880011.390239:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880011.390243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880011.390251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880011.390271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880011.390282:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880011/real 1788880011] req@ffffa061888ad500 x1875771511695744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880011.390305:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511695744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880011.390314:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880011.390318:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880011.390322:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880011.390328:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880011.390348:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880011.390358:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880011/real 1788880011] req@ffffa061888ac000 x1875771511695872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880011.390372:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511695872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880011.390413:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880011.390418:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880011.390422:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511695872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880011.390430:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880011.390453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880011.390467:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880011/real 1788880011] req@ffffa061888adf80 x1875771511696000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880011.390482:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511696000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880011.390500:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880011.390504:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880011.390508:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880012.029507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880012.029618:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880012.285486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880013.053522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880013.053648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880013.309496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880014.077496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880014.077704:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880014.333489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880015.101456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880015.101514:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880015.357505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880016.125463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880016.125513:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880016.381458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880016.509541:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880016.509550:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880016.509553:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880016.509563:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5003 00000100:00080000:1.0:1788880016.509573:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880016.509580:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880016.509614:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511696128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880016.509638:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880016.509640:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880016.509642:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880016.509645:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5003 00000100:00080000:1.0:1788880016.509647:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880016.509651:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880016.509660:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880016.509671:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511696128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880016.509674:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511696256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880016.509685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880016.509688:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880016.509689:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880016.509691:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5003 00000100:00080000:1.0:1788880016.509693:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880016.509695:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880016.509698:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880016.509705:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511696384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880016.509722:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880016.509725:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880016.509726:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880016.509729:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5003 00000100:00080000:1.0:1788880016.509744:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880016.509746:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880016.509750:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880016.509754:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880016.509762:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511696512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880016.509769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880016.509772:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880016.509773:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880016.509776:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5003 00000100:00080000:1.0:1788880016.509778:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880016.509781:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880016.509784:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880016.509790:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880016/real 1788880016] req@ffffa0618869ad80 x1875771511696128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880016.509794:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511696640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880016.509833:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511696128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880016.509842:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880016.509845:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880016.509848:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511696128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880016.509866:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880016.509881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880016.509889:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880016/real 1788880016] req@ffffa0618869bb80 x1875771511696256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880016.509897:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511696256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880016.509902:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880016.509905:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880016.509908:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880016.509912:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880016.509923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880016.509929:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880016/real 1788880016] req@ffffa0618869a680 x1875771511696384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880016.509936:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511696384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880016.509941:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880016.509943:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880016.509945:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880016.509948:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880016.509959:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880016.509965:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880016/real 1788880016] req@ffffa06188698000 x1875771511696512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880016.509973:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511696512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880016.509978:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880016.509980:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880016.509982:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880016.509985:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880016.509998:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880016.510009:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880016/real 1788880016] req@ffffa06188699f80 x1875771511696640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880016.510021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511696640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880016.510030:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880016.510033:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880016.510037:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880017.149473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880017.149531:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880017.405461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880018.173492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880018.173548:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880018.429470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880019.197446:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880019.197523:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880019.453465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880020.221490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880020.221611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880020.477497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880021.245448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880021.245503:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880021.501505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880021.629605:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880021.629616:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880021.629623:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880021.629641:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5008 00000100:00080000:2.0:1788880021.629658:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880021.629669:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880021.629722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511696768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880021.629752:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880021.629759:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880021.629762:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880021.629772:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5008 00000100:00080000:2.0:1788880021.629782:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880021.629789:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880021.629804:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880021.629830:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511696896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880021.629835:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511696768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880021.629861:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880021.629872:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880021.629875:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880021.629882:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5008 00000100:00080000:2.0:1788880021.629890:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880021.629899:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880021.629910:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880021.629942:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511697024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880021.629966:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880021.629973:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880021.629979:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880021.629991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880021.629999:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5008 00000100:00080000:2.0:1788880021.630012:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880021.630047:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880021.630062:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880021.630094:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511697152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880021.630101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880021/real 1788880021] req@ffffa061888aca80 x1875771511696768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880021.630176:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880021.630184:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880021.630187:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880021.630195:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511696768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880021.630196:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5008 00000100:00080000:2.0:1788880021.630213:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880021.630235:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788880021.630243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788880021.630247:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880021.630269:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880021.630281:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511697280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880021.630287:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511696768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880021.630368:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880021.630450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880021.630479:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880021/real 1788880021] req@ffffa061888af480 x1875771511696896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880021.630523:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511696896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880021.630542:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880021.630547:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880021.630552:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511696896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880021.630562:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880021.630594:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880021.630609:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880021/real 1788880021] req@ffffa061888adf80 x1875771511697024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880021.630624:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511697024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880021.630635:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880021.630639:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880021.630643:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880021.630651:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880021.630672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880021.630684:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880021/real 1788880021] req@ffffa061888ac000 x1875771511697152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880021.630698:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511697152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880021.630717:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880021.630721:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880021.630725:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880021.630733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880021.630752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880021.630763:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880021/real 1788880021] req@ffffa061888ad500 x1875771511697280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880021.630777:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511697280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880021.630788:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880021.630792:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880021.630795:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880022.269458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880022.269543:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880022.525547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880023.293448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880023.293539:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880023.549462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880024.317527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880024.317637:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880024.573600:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880025.341480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880025.341625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880025.597607:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880026.365522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880026.365656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880026.621505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880026.749529:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880026.749543:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880026.749550:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880026.749566:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5013 00000100:00080000:1.0:1788880026.749581:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880026.749592:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880026.749646:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511697408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880026.749679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880026.749683:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880026.749685:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880026.749690:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5013 00000100:00080000:1.0:1788880026.749695:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880026.749701:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880026.749714:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880026.749726:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511697408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880026.749734:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511697536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880026.749755:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880026.749758:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880026.749760:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880026.749764:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5013 00000100:00080000:1.0:1788880026.749768:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880026.749772:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880026.749777:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880026.749788:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511697664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880026.749798:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880026.749802:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880026.749803:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880026.749807:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5013 00000100:00080000:1.0:1788880026.749811:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880026.749816:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880026.749838:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880026.749849:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880026.749868:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511697792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880026.749881:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880026.749886:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880026.749889:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880026.749897:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5013 00000100:00100000:0.0:1788880026.749898:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880026/real 1788880026] req@ffffa0618869b100 x1875771511697408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880026.749906:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880026.749920:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880026.749931:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880026.749943:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511697408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880026.749950:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511697920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880026.750026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880026.750035:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880026.750041:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511697408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880026.750089:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880026.750123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880026.750140:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880026/real 1788880026] req@ffffa06188698380 x1875771511697536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880026.750156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511697536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880026.750167:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880026.750171:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880026.750175:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880026.750182:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880026.750202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880026.750213:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880026/real 1788880026] req@ffffa0618869b800 x1875771511697664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880026.750227:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511697664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880026.750236:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880026.750240:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880026.750244:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880026.750251:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880026.750271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880026.750282:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880026/real 1788880026] req@ffffa06188699c00 x1875771511697792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880026.750296:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511697792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880026.750306:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880026.750309:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880026.750313:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880026.750319:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880026.750352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880026.750361:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880026/real 1788880026] req@ffffa0618869aa00 x1875771511697920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880026.750370:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511697920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880026.750377:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880026.750380:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880026.750383:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511697920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880027.389497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880027.389660:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880027.645482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880028.413489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880028.413682:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880028.669486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880029.437514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880029.437653:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880029.693504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880030.461514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880030.461632:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880030.717485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880031.485497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880031.485618:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880031.741504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880031.869495:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880031.869507:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880031.869513:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880031.869528:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5018 00000100:00080000:1.0:1788880031.869546:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880031.869557:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880031.869609:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511698048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880031.869645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880031.869650:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880031.869653:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880031.869658:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5018 00000100:00080000:1.0:1788880031.869663:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880031.869668:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880031.869681:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880031.869699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511698176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880031.869715:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511698048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880031.869718:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880031.869730:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880031.869735:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880031.869741:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5018 00000100:00080000:1.0:1788880031.869744:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880031.869748:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880031.869754:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880031.869770:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511698304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880031.869782:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880031.869787:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880031.869790:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880031.869795:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5018 00000100:00080000:1.0:1788880031.869813:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880031.869817:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880031.869824:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880031.869826:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880031.869848:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511698432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880031.869862:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880031.869867:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880031.869869:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880031.869876:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5018 00000100:00080000:1.0:1788880031.869883:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880031.869884:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880031/real 1788880031] req@ffffa0618869aa00 x1875771511698048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880031.869899:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880031.869916:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880031.869969:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511698048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880031.869974:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511698560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880031.870003:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880031.870016:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880031.870024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511698048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880031.870056:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880031.870091:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880031.870110:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880031/real 1788880031] req@ffffa06188699c00 x1875771511698176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880031.870125:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511698176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880031.870136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880031.870140:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880031.870144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880031.870151:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880031.870171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880031.870183:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880031/real 1788880031] req@ffffa0618869b800 x1875771511698304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880031.870197:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511698304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880031.870215:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880031.870219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880031.870223:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880031.870230:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880031.870249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880031.870261:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880031/real 1788880031] req@ffffa06188698380 x1875771511698432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880031.870290:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511698432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880031.870301:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880031.870305:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880031.870309:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880031.870316:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880031.870336:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880031.870348:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880031/real 1788880031] req@ffffa0618869b100 x1875771511698560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880031.870362:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511698560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880031.870371:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880031.870374:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880031.870380:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880032.509515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880032.509650:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880032.765504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880033.533527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880033.533604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880033.789508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880034.557515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880034.557678:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880034.813510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880035.581519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880035.581652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880035.837506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880036.605509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880036.605669:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880036.861517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880036.989566:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880036.989577:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880036.989583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880036.989599:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5023 00000100:00080000:2.0:1788880036.989614:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880036.989625:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880036.989676:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511698688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880036.989714:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880036.989724:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880036.989727:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880036.989738:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5023 00000100:00080000:2.0:1788880036.989744:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880036.989751:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880036.989766:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880036.989786:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511698688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880036.989790:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511698816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880036.989813:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880036.989817:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880036.989820:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880036.989825:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5023 00000100:00080000:2.0:1788880036.989831:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880036.989835:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880036.989841:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880036.989858:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511698944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880036.989870:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880036.989875:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880036.989878:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880036.989881:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5023 00000100:00080000:2.0:1788880036.989887:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880036.989891:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880036.989898:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880036.989948:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511699072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880036.989966:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880036.989973:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698816:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788880036.989974:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880036.989982:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880036.989995:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5023 00000100:00080000:2.0:1788880036.990003:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880036.990010:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880036.990020:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880036.990037:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880036.990048:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511699200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880036.990089:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880036.990121:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880037.629540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880037.629658:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880037.885529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880038.653522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880038.653668:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880038.909534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880039.677499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880039.677658:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880039.933503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880040.061650:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880040.061687:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880040.061700:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880040.061731:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880040.061737:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880040.061743:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880040.061751:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788880040.061803:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880036/real 1788880040] req@ffffa061888afb80 x1875771511699072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880040.061845:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511699072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880040.061864:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880040.061874:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880040.061902:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880040.061920:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880036/real 1788880040] req@ffffa061888af100 x1875771511699200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880040.061946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511699200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880040.061975:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880040.061983:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880040.061990:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880040.062005:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880036/real 1788880040] req@ffffa061888ae300 x1875771511698816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880040.062031:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511698816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880040.062047:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880040.062054:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880040.062060:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880040.062074:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880036/real 1788880040] req@ffffa061888aea00 x1875771511698944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880040.062100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511698944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880040.062116:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880040.062124:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880040.062131:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511698944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880040.062146:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880036/real 1788880040] req@ffffa061888adc00 x1875771511698688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880040.062170:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511698688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880040.062186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880040.062192:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880040.062199:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511698688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880040.701480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880040.701608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880040.957502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880041.725499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880041.725673:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880041.981479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880042.109487:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880042.109498:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880042.109504:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880042.109517:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5028 00000100:00080000:2.0:1788880042.109532:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880042.109543:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880042.109595:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511699328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880042.109654:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880042.109660:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880042.109663:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880042.109669:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5028 00000100:00080000:2.0:1788880042.109674:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880042.109680:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880042.109693:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880042.109709:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511699456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880042.109722:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880042.109726:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880042.109729:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880042.109732:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5028 00000100:00080000:2.0:1788880042.109736:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880042.109741:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880042.109746:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880042.109759:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511699584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880042.109772:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880042.109776:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880042.109779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880042.109791:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5028 00000100:00100000:1.0:1788880042.109795:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511699328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880042.109797:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880042.109810:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880042.109823:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880042.109851:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511699712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880042.109894:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880042.109901:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880042.109905:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880042.109913:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5028 00000100:00080000:2.0:1788880042.109923:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788880042.109934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880042.109938:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880042.109953:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880042.109993:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511699840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788880042.110009:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880042/real 1788880042] req@ffffa061888aed80 x1875771511699328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880042.110073:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511699328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880042.110094:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880042.110102:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788880042.110110:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511699328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880042.110126:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788880042.110173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880042.110198:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880042/real 1788880042] req@ffffa061888ac700 x1875771511699456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880042.110223:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511699456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880042.110240:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880042.110246:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788880042.110253:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788880042.110266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:1.0:1788880042.110309:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880042.110334:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880042/real 1788880042] req@ffffa061888ac380 x1875771511699584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880042.110361:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511699584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880042.110430:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880042.110439:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788880042.110447:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788880042.110495:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788880042.110541:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880042.110560:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880042/real 1788880042] req@ffffa061888ad880 x1875771511699712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880042.110576:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511699712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880042.110587:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880042.110591:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788880042.110596:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880042.110603:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:1.0:1788880042.110630:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880042.110661:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880042/real 1788880042] req@ffffa061888ae680 x1875771511699840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880042.110689:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511699840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880042.110705:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880042.110712:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:1.0:1788880042.110720:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511699840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880042.749524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880042.749610:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880043.005538:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880043.773498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880043.773656:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880044.029502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880044.797497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880044.797658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880045.053487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880045.821509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880045.821696:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880046.077470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880046.845519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880046.845607:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880047.101481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880047.229692:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880047.229702:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880047.229708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880047.229723:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5033 00000100:00080000:1.0:1788880047.229738:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880047.229750:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880047.229796:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511699968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880047.229830:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880047.229861:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880047.229864:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880047.229872:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5033 00000100:00080000:1.0:1788880047.229878:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880047.229884:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880047.229898:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880047.229902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511699968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880047.229919:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511700096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880047.229937:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880047.229942:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880047.229946:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880047.229951:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5033 00000100:00080000:1.0:1788880047.229981:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880047.229987:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880047.229994:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788880047.229998:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880047.230017:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511700224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880047.230031:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880047.230036:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880047.230038:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880047.230043:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5033 00000100:00080000:1.0:1788880047.230049:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880047.230054:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880047/real 1788880047] req@ffffa06188699f80 x1875771511699968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880047.230060:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880047.230074:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880047.230097:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511700352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880047.230104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511699968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880047.230126:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880047.230136:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880047.230140:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880047.230141:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880047.230150:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5033 00000100:00080000:0.0:1788880047.230155:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880047.230164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511699968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880047.230170:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880047.230179:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880047.230185:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880047.230200:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511700480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880047.230212:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880047.230269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880047.230296:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880047/real 1788880047] req@ffffa06188698000 x1875771511700096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880047.230317:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511700096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880047.230329:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880047.230333:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880047.230338:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880047.230346:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880047.230373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880047.230420:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880047/real 1788880047] req@ffffa0618869a680 x1875771511700224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880047.230435:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511700224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880047.230445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880047.230449:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880047.230453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880047.230460:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880047.230483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880047.230496:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880047/real 1788880047] req@ffffa0618869bb80 x1875771511700352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880047.230509:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511700352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880047.230519:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880047.230523:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880047.230526:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880047.230547:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880047.230567:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880047.230579:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880047/real 1788880047] req@ffffa0618869ad80 x1875771511700480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880047.230593:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511700480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880047.230603:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880047.230606:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880047.230610:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880047.869540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880047.869629:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880048.125501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880048.893534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880048.893657:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880049.149511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880049.917489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880049.917668:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880050.173528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880050.941612:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880050.941649:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880051.197502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880051.965616:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880051.965662:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880052.221511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880052.349536:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880052.349547:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880052.349552:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880052.349566:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5039 00000100:00080000:1.0:1788880052.349586:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880052.349601:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880052.349657:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511700608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880052.349701:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880052.349708:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880052.349710:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880052.349716:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5039 00000100:00080000:1.0:1788880052.349721:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880052.349725:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880052.349738:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880052.349753:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511700736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880052.349766:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880052.349770:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880052.349774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880052.349780:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5039 00000100:00080000:1.0:1788880052.349786:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880052.349791:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880052.349824:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880052.349847:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511700608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880052.349848:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511700864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880052.349875:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880052.349880:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880052.349883:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880052.349890:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5039 00000100:00080000:1.0:1788880052.349894:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880052.349898:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880052.349906:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880052.349922:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511700992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880052.349935:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880052.349940:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880052.349942:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880052.349946:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5039 00000100:00080000:1.0:1788880052.349952:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788880052.349959:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880052.349965:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880052.349975:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880052.349994:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511701120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880052.350022:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880052/real 1788880052] req@ffffa0618869ad80 x1875771511700608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880052.350052:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511700608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880052.350083:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880052.350089:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880052.350095:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511700608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880052.350105:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880052.350135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880052.350151:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880052/real 1788880052] req@ffffa0618869bb80 x1875771511700736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880052.350167:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511700736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880052.350178:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880052.350182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880052.350188:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880052.350216:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880052.350238:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880052.350251:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880052/real 1788880052] req@ffffa0618869a680 x1875771511700864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880052.350265:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511700864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880052.350275:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880052.350278:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880052.350282:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880052.350289:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880052.350308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880052.350318:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880052/real 1788880052] req@ffffa06188698000 x1875771511700992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880052.350333:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511700992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880052.350343:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880052.350346:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880052.350349:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511700992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880052.350356:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880052.350373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880052.350453:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880052/real 1788880052] req@ffffa06188699f80 x1875771511701120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880052.350480:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511701120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880052.350497:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880052.350504:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880052.350511:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880052.989523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880052.989593:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880053.245459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880054.013553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880054.013659:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880054.269545:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880055.037533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880055.037558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880055.293524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880056.061525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880056.061605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880056.317503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880057.085618:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880057.085707:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880057.341481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880057.469575:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880057.469585:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880057.469592:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880057.469611:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5044 00000100:00080000:1.0:1788880057.469628:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880057.469642:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880057.469695:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511701248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880057.469733:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880057.469739:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880057.469741:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880057.469747:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5044 00000100:00080000:1.0:1788880057.469752:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880057.469756:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880057.469793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880057.469811:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511701376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880057.469825:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880057.469831:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880057.469834:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880057.469839:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5044 00000100:00080000:1.0:1788880057.469844:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880057.469850:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880057.469856:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880057.469869:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511701504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880057.469880:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511701248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880057.469887:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880057.469900:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880057.469903:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880057.469908:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5044 00000100:00080000:1.0:1788880057.469912:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880057.469916:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880057.469924:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880057.469940:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511701632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880057.469952:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880057.469957:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880057.469960:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880057.469964:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5044 00000100:00080000:1.0:1788880057.469969:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880057.469973:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880057.469980:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880057.469996:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511701760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880057.470002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880057.470066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880057/real 1788880057] req@ffffa0618869b100 x1875771511701248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880057.470093:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511701248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880057.470107:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880057.470113:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880057.470119:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511701248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880057.470130:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880057.470160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880057.470175:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880057/real 1788880057] req@ffffa06188698380 x1875771511701376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880057.470190:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511701376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880057.470201:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880057.470204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880057.470208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880057.470237:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880057.470258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880057.470270:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880057/real 1788880057] req@ffffa0618869b800 x1875771511701504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880057.470284:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511701504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880057.470294:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880057.470298:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880057.470302:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880057.470309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880057.470329:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880057.470340:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880057/real 1788880057] req@ffffa06188699c00 x1875771511701632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880057.470355:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511701632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880057.470365:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880057.470368:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880057.470372:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880057.470378:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880057.470455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880057.470471:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880057/real 1788880057] req@ffffa0618869aa00 x1875771511701760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880057.470486:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511701760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880057.470497:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880057.470501:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880057.470504:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511701760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880058.109570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880058.109643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880058.365511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880059.133534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880059.133669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880059.389481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880060.157541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880060.157656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880060.413520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880061.181529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880061.181725:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880061.437518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880062.205566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880062.205639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880062.461549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880062.589531:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880062.589542:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880062.589549:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880062.589564:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5049 00000100:00080000:1.0:1788880062.589581:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880062.589592:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880062.589642:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511701888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880062.589679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880062.589684:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880062.589687:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880062.589692:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5049 00000100:00080000:1.0:1788880062.589696:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880062.589701:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880062.589714:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880062.589756:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511702016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880062.589770:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880062.589775:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880062.589778:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880062.589783:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5049 00000100:00080000:1.0:1788880062.589788:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788880062.589797:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511701888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880062.589798:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880062.589810:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880062.589826:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511702144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880062.589840:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880062.589845:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880062.589847:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880062.589852:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5049 00000100:00080000:1.0:1788880062.589856:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880062.589860:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880062.589867:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880062.589882:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511702272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880062.589894:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880062.589899:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880062.589901:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880062.589904:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5049 00000100:00080000:1.0:1788880062.589909:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788880062.589916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880062.589921:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880062.589929:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880062.589945:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511702400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880062.589984:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880062/real 1788880062] req@ffffa0618869aa00 x1875771511701888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880062.590012:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511701888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880062.590027:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880062.590032:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880062.590038:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511701888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880062.590047:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880062.590077:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880062.590092:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880062/real 1788880062] req@ffffa06188699c00 x1875771511702016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880062.590107:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511702016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880062.590117:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880062.590121:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880062.590125:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880062.590155:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880062.590176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880062.590189:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880062/real 1788880062] req@ffffa0618869b800 x1875771511702144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880062.590204:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511702144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880062.590214:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880062.590217:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880062.590221:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880062.590228:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880062.590248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880062.590259:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880062/real 1788880062] req@ffffa06188698380 x1875771511702272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880062.590278:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511702272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880062.590294:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880062.590300:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880062.590306:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880062.590317:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880062.590357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880062.590436:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880062/real 1788880062] req@ffffa0618869b100 x1875771511702400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880062.590457:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511702400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880062.590468:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880062.590472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880062.590476:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880063.229530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880063.229630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880063.485525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880064.253515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880064.253618:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880064.509500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880065.277506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880065.277638:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880065.533511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880066.301595:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880066.301683:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880066.557516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880067.325555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880067.325638:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880067.581503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880067.709616:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880067.709628:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880067.709634:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880067.709649:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5054 00000100:00080000:2.0:1788880067.709665:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880067.709675:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880067.709748:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511702528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880067.709798:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880067.709804:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880067.709806:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880067.709812:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5054 00000100:00080000:2.0:1788880067.709818:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880067.709825:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880067.709839:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880067.709859:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511702656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880067.709882:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880067.709883:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511702528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880067.709895:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880067.709905:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880067.709931:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5054 00000100:00080000:2.0:1788880067.709936:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880067.709940:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880067.709947:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880067.709967:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511702784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880067.709980:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880067.709985:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880067.709988:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880067.709993:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5054 00000100:00080000:2.0:1788880067.709997:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880067.710002:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880067.710011:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880067.710023:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511702912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880067.710046:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788880067.710061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880067.710061:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880067.710078:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880067.710085:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5054 00000100:00080000:2.0:1788880067.710090:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880067.710095:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880067.710102:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880067.710122:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511703040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880067.710178:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880067/real 1788880067] req@ffffa061888ace00 x1875771511702528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880067.710209:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511702528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880067.710226:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880067.710232:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880067.710238:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511702528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880067.710276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880067.710312:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880067.710328:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880067/real 1788880067] req@ffffa061888ad180 x1875771511702656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880067.710344:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511702656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880067.710355:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880067.710359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880067.710363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880067.710371:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880067.710436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880067.710452:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880067/real 1788880067] req@ffffa061888ae680 x1875771511702784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880067.710477:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511702784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880067.710488:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880067.710492:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880067.710496:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880067.710503:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880067.710526:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880067.710538:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880067/real 1788880067] req@ffffa061888ad880 x1875771511702912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880067.710552:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511702912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880067.710562:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880067.710565:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880067.710569:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511702912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880067.710577:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880067.710596:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880067.710608:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880067/real 1788880067] req@ffffa061888ac380 x1875771511703040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880067.710622:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511703040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880067.710640:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880067.710645:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880067.710649:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880068.349545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880068.349638:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880068.605513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880069.373478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880069.373653:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880069.629514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880070.397507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880070.397636:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880070.653479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880071.421490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880071.421658:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880071.677484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880072.445526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880072.445597:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880072.701475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880072.829502:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880072.829514:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880072.829520:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880072.829536:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5059 00000100:00080000:2.0:1788880072.829554:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880072.829564:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880072.829616:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511703168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880072.829674:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880072.829680:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880072.829683:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880072.829690:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5059 00000100:00080000:2.0:1788880072.829695:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880072.829700:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880072.829714:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880072.829733:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511703296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880072.829748:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880072.829754:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511703168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880072.829755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880072.829763:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880072.829775:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5059 00000100:00080000:2.0:1788880072.829779:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880072.829784:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880072.829791:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880072.829806:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511703424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880072.829820:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880072.829825:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880072.829828:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880072.829833:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5059 00000100:00080000:2.0:1788880072.829838:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880072.829844:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880072.829851:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880072.829862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880072.829870:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511703552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880072.829885:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880072.829891:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880072.829893:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880072.829898:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5059 00000100:00080000:2.0:1788880072.829904:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880072.829910:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880072.829916:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880072.829925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880072/real 1788880072] req@ffffa061888ac700 x1875771511703168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880072.829933:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511703680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880072.829984:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511703168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880072.830001:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880072.830007:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880072.830013:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511703168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880072.830042:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880072.830070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880072.830085:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880072/real 1788880072] req@ffffa061888aed80 x1875771511703296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880072.830100:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511703296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880072.830111:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880072.830115:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880072.830120:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880072.830127:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880072.830147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880072.830158:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880072/real 1788880072] req@ffffa061888adc00 x1875771511703424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880072.830172:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511703424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880072.830182:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880072.830186:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880072.830190:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880072.830197:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880072.830232:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880072.830245:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880072/real 1788880072] req@ffffa061888aea00 x1875771511703552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880072.830259:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511703552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880072.830269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880072.830272:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880072.830277:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880072.830285:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880072.830303:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880072.830314:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880072/real 1788880072] req@ffffa061888ae300 x1875771511703680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880072.830328:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511703680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880072.830338:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880072.830341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880072.830344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880073.469517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880073.469611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880073.725475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880074.493476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880074.493671:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880074.749487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880075.517535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880075.517627:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880075.773499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880076.541537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880076.541612:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880076.797503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880077.565534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880077.565677:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880077.821466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880077.949527:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880077.949538:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880077.949543:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880077.949557:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5064 00000100:00080000:0.0:1788880077.949571:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880077.949580:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880077.949651:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511703808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880077.949688:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880077.949693:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880077.949696:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880077.949701:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5064 00000100:00080000:0.0:1788880077.949706:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880077.949710:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880077.949723:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880077.949740:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511703936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880077.949753:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880077.949757:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880077.949760:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880077.949765:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5064 00000100:00080000:0.0:1788880077.949769:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880077.949774:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880077.949784:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511703808:192.168.203.111@tcp:250:kworker.0 00000008:00100000:0.0:1788880077.949785:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880077.949806:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511704064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880077.949820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880077.949825:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880077.949827:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880077.949831:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5064 00000100:00080000:0.0:1788880077.949836:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880077.949840:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880077.949848:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880077.949863:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511704192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880077.949875:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880077.949880:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880077.949882:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788880077.949892:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788880077.949893:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5064 00000100:00080000:0.0:1788880077.949905:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880077.949910:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880077.949918:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880077.949932:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511704320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880077.949953:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880077/real 1788880077] req@ffffa06198afe300 x1875771511703808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880077.949990:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511703808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880077.950008:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880077.950017:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880077.950026:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511703808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880077.950042:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880077.950076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880077.950098:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880077/real 1788880077] req@ffffa06198afc380 x1875771511703936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880077.950113:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511703936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880077.950124:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880077.950129:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880077.950133:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511703936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880077.950163:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880077.950183:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880077.950195:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880077/real 1788880077] req@ffffa06198afca80 x1875771511704064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880077.950219:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511704064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880077.950229:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880077.950232:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880077.950236:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880077.950243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880077.950272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880077.950293:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880077/real 1788880077] req@ffffa06198afd500 x1875771511704192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880077.950318:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511704192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880077.950335:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880077.950341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880077.950348:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880077.950361:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880077.950423:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880077.950441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880077/real 1788880077] req@ffffa06198afd180 x1875771511704320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880077.950456:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511704320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880077.950474:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880077.950478:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880077.950482:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880078.589524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880078.589677:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880078.845482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880079.613533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880079.613642:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880079.869509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880080.637535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880080.637688:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880080.893488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880081.661487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880081.661678:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880081.917494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880082.685479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880082.685623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880082.941510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880083.069506:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880083.069516:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880083.069521:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880083.069534:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5069 00000100:00080000:0.0:1788880083.069548:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880083.069558:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880083.069640:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511704448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880083.069686:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880083.069690:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880083.069693:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880083.069705:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5069 00000100:00080000:0.0:1788880083.069711:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880083.069717:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880083.069730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880083.069747:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511704576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880083.069762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880083.069770:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1788880083.069773:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511704448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880083.069774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880083.069787:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5069 00000100:00080000:0.0:1788880083.069792:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880083.069796:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880083.069803:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880083.069818:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511704704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880083.069830:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880083.069836:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880083.069838:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880083.069844:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5069 00000100:00080000:0.0:1788880083.069849:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880083.069853:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880083.069859:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880083.069875:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511704832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880083.069884:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788880083.069901:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880083.069906:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880083.069908:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880083.069913:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5069 00000100:00080000:0.0:1788880083.069917:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880083.069921:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880083.069927:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880083.069940:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511704960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880083.069973:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880083/real 1788880083] req@ffffa06198afd180 x1875771511704448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880083.070022:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511704448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880083.070045:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880083.070051:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880083.070058:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511704448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880083.070094:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880083.070131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880083.070152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880083/real 1788880083] req@ffffa06198afd500 x1875771511704576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880083.070170:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511704576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880083.070182:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880083.070188:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880083.070193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880083.070205:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880083.070227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880083.070239:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880083/real 1788880083] req@ffffa06198afca80 x1875771511704704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880083.070253:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511704704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880083.070262:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880083.070265:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880083.070269:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880083.070277:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880083.070296:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880083.070307:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880083/real 1788880083] req@ffffa06198afc380 x1875771511704832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880083.070321:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511704832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880083.070332:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880083.070336:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880083.070340:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880083.070347:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880083.070365:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880083.070375:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880083/real 1788880083] req@ffffa06198afe300 x1875771511704960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880083.070430:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511704960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880083.070440:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880083.070444:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880083.070448:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511704960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880083.709524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880083.709646:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880083.965504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880084.733483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880084.733636:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880084.989532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880085.757471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880085.757543:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880086.013455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880086.781480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880086.781547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880087.037475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880087.805465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880087.805533:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880088.061444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880088.189489:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880088.189500:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880088.189504:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880088.189516:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5074 00000100:00080000:2.0:1788880088.189527:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880088.189535:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880088.189589:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511705088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880088.189618:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880088.189623:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880088.189625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880088.189631:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5074 00000100:00080000:2.0:1788880088.189636:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880088.189641:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880088.189652:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880088.189655:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511705088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880088.189669:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511705216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880088.189683:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880088.189687:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880088.189690:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880088.189695:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5074 00000100:00080000:2.0:1788880088.189700:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880088.189705:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880088.189711:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788880088.189720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880088.189726:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511705344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880088.189744:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880088.189748:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880088.189750:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880088.189757:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5074 00000100:00100000:0.0:1788880088.189758:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880088/real 1788880088] req@ffffa061888af100 x1875771511705088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880088.189762:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880088.189771:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880088.189779:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880088.189783:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511705088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880088.189795:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788880088.189796:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511705472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880088.189802:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880088.189810:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511705088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880088.189817:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880088.189823:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880088.189825:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880088.189830:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5074 00000100:00100000:0.0:1788880088.189835:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705216:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788880088.189835:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880088.189843:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880088.189850:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880088.189861:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880088.189867:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511705600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880088.189873:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880088/real 1788880088] req@ffffa061888afb80 x1875771511705216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880088.189892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511705216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880088.189899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880088.189902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880088.189905:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880088.189910:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880088.189922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880088.189929:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880088/real 1788880088] req@ffffa061888ad500 x1875771511705344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880088.189936:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511705344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880088.189942:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880088.189943:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880088.189945:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880088.189950:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880088.189960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880088.189965:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880088/real 1788880088] req@ffffa061888ac000 x1875771511705472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880088.189973:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511705472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880088.189979:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880088.189980:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880088.189982:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880088.189991:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880088.190000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880088.190006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880088/real 1788880088] req@ffffa061888adf80 x1875771511705600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880088.190013:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511705600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880088.190018:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880088.190020:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880088.190022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880088.829479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880088.829575:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880089.085475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880089.853453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880089.853517:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880090.109455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880090.877559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880090.877629:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880091.133500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880091.901473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880091.901539:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880092.157543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880092.925582:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880092.925602:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880093.181471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880093.309525:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880093.309535:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880093.309539:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880093.309549:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5080 00000100:00080000:1.0:1788880093.309561:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880093.309570:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880093.309607:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511705728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880093.309633:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880093.309636:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880093.309637:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880093.309641:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5080 00000100:00080000:1.0:1788880093.309644:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880093.309647:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880093.309657:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880093.309665:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511705728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880093.309669:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511705856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880093.309679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880093.309681:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880093.309683:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880093.309685:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5080 00000100:00080000:1.0:1788880093.309687:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880093.309689:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880093.309692:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880093.309699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511705984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880093.309706:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880093.309710:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880093.309711:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880093.309714:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5080 00000100:00080000:1.0:1788880093.309727:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880093.309730:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880093.309733:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880093.309739:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880093.309743:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511706112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880093.309750:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880093.309753:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880093.309754:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880093.309757:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5080 00000100:00080000:1.0:1788880093.309760:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880093.309762:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880093.309765:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880093.309772:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511706240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880093.309778:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880093/real 1788880093] req@ffffa06188699f80 x1875771511705728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880093.309796:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511705728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880093.309805:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880093.309808:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880093.309812:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511705728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880093.309834:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880093.309849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880093.309856:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880093/real 1788880093] req@ffffa06188698000 x1875771511705856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880093.309864:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511705856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880093.309870:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880093.309873:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880093.309875:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880093.309878:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880093.309888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880093.309893:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880093/real 1788880093] req@ffffa0618869a680 x1875771511705984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880093.309902:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511705984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880093.309907:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880093.309909:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880093.309911:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511705984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880093.309914:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880093.309925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880093.309931:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880093/real 1788880093] req@ffffa0618869bb80 x1875771511706112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880093.309939:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511706112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880093.309944:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880093.309946:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880093.309948:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880093.309951:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880093.309961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880093.309966:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880093/real 1788880093] req@ffffa0618869ad80 x1875771511706240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880093.309974:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511706240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880093.309984:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880093.309986:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880093.309988:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880093.949536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880093.949612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880094.205460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880094.973479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880094.973562:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880095.229449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880095.997571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880095.997604:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880096.253535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880097.021606:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880097.021750:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880097.277524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880098.045565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880098.045684:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880098.301571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880098.429584:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880098.429602:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880098.429610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880098.429629:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5085 00000100:00080000:1.0:1788880098.429648:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880098.429662:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880098.429727:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511706368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880098.429781:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880098.429789:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880098.429794:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880098.429803:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5085 00000100:00080000:1.0:1788880098.429813:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880098.429824:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880098.429843:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880098.429846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511706368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880098.429880:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511706496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880098.429908:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880098.429916:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880098.429920:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880098.429929:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5085 00000100:00080000:1.0:1788880098.429939:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880098.429949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880098.429958:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880098.429972:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880098.430002:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880098/real 1788880098] req@ffffa0618869ad80 x1875771511706368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880098.430007:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511706624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880098.430057:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880098.430059:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511706368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880098.430077:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880098.430087:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880098.430088:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880098.430104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788880098.430112:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5085 00000100:00100000:0.0:1788880098.430117:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511706368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880098.430132:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880098.430148:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880098.430160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706496:192.168.203.111@tcp:38:kworker.0 00000008:00100000:1.0:1788880098.430162:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880098.430203:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511706752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880098.430208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880098.430231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880098/real 1788880098] req@ffffa0618869bb80 x1875771511706496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880098.430238:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880098.430260:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880098.430269:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880098.430270:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511706496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880098.430290:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5085 00000100:00080000:0.0:1788880098.430295:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880098.430305:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788880098.430307:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880098.430313:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706496:192.168.203.111@tcp:38:kworker.0 00000100:00080000:1.0:1788880098.430327:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880098.430329:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706624:192.168.203.111@tcp:38:kworker.0 00000008:00100000:1.0:1788880098.430351:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880098.430366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880098.430416:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880098/real 1788880098] req@ffffa0618869a680 x1875771511706624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880098.430435:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511706624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880098.430447:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511706880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880098.430450:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880098.430461:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880098.430500:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880098.430543:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880098.430579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880098.430594:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880098/real 1788880098] req@ffffa06188698000 x1875771511706752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880098.430608:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511706752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880098.430619:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880098.430623:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880098.430627:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880098.430634:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880098.430653:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880098.430664:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880098/real 1788880098] req@ffffa06188699f80 x1875771511706880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880098.430678:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511706880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880098.430688:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880098.430693:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880098.430699:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511706880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880099.069520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880099.069660:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880099.325568:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880100.093510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880100.093636:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880100.349493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880101.117543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880101.117694:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880101.373533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880102.141539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880102.141661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880102.397472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880103.165547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880103.165617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880103.421522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880103.549592:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880103.549602:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880103.549609:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880103.549624:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5090 00000100:00080000:2.0:1788880103.549639:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880103.549650:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880103.549706:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511707008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880103.549745:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880103.549751:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880103.549754:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880103.549759:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5090 00000100:00080000:2.0:1788880103.549764:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880103.549770:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880103.549785:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880103.549810:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511707008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880103.549810:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511707136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880103.549840:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880103.549844:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880103.549847:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880103.549853:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5090 00000100:00080000:2.0:1788880103.549857:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880103.549861:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880103.549868:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880103.549885:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511707264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880103.549898:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880103.549905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880103.549908:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880103.549917:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5090 00000100:00080000:2.0:1788880103.549925:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880103.549934:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880103.549946:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880103.549972:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511707392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880103.549975:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707136:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788880103.550010:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880103.550015:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880103.550018:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880103.550025:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5090 00000100:00080000:2.0:1788880103.550034:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880103.550046:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880103.550058:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880103.550062:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880103.550083:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511707520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880103.550120:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880103.550152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880104.189538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880104.189646:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880104.445564:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880105.213514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880105.213700:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880105.469563:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880106.237533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880106.237632:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880106.493519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880106.621801:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880106.621882:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880106.621914:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880106.621993:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880106.622014:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880106.622035:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880106.622055:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788880106.622097:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880103/real 1788880106] req@ffffa061888ac000 x1875771511707392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880106.622151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511707392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880106.622188:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880106.622199:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880106.622208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880106.622225:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880103/real 1788880106] req@ffffa061888ad500 x1875771511707520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880106.622250:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511707520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880106.622268:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880106.622275:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880106.622282:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880106.622295:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880103/real 1788880106] req@ffffa061888aca80 x1875771511707136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880106.622321:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511707136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880106.622341:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880106.622349:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880106.622358:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880106.622372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880103/real 1788880106] req@ffffa061888adf80 x1875771511707264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880106.622469:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511707264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880106.622488:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880106.622494:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880106.622501:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880106.622518:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880103/real 1788880106] req@ffffa061888af480 x1875771511707008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880106.622543:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511707008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880106.622558:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880106.622566:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880106.622571:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511707008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880107.261508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880107.261631:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880107.517531:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880108.285529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880108.285627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880108.541498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880108.669587:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880108.669604:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880108.669613:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880108.669635:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5095 00000100:00080000:0.0:1788880108.669655:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880108.669668:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880108.669740:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511707648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880108.669780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880108.669785:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880108.669788:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880108.669794:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5095 00000100:00080000:0.0:1788880108.669799:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880108.669804:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880108.669827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880108.669848:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511707648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880108.669850:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511707776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880108.669878:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880108.669882:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880108.669884:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880108.669889:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5095 00000100:00080000:0.0:1788880108.669892:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880108.669896:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880108.669902:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880108.669925:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511707904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880108.669937:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880108.669942:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880108.669945:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880108.669951:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5095 00000100:00080000:0.0:1788880108.669966:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880108.669972:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788880108.669981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:0.0:1788880108.669988:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880108.670019:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511708032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880108.670041:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880108.670047:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880108.670054:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880108.670058:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880108/real 1788880108] req@ffffa06198afea00 x1875771511707648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880108.670068:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5095 00000100:00080000:0.0:1788880108.670084:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880108.670099:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880108.670110:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880108.670115:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511707648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880108.670140:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511708160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880108.670143:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880108.670164:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880108.670182:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511707648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880108.670241:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880108.670321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880108.670354:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880108/real 1788880108] req@ffffa06198afc000 x1875771511707776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880108.670458:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511707776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880108.670479:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880108.670486:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880108.670493:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880108.670506:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880108.670554:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880108.670583:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880108/real 1788880108] req@ffffa06198afc700 x1875771511707904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880108.670621:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511707904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880108.670638:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880108.670644:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880108.670651:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511707904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880108.670664:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880108.670706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880108.670730:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880108/real 1788880108] req@ffffa06198afdc00 x1875771511708032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880108.670754:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511708032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880108.670770:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880108.670777:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880108.670784:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880108.670798:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880108.670834:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880108.670851:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880108/real 1788880108] req@ffffa06198afdf80 x1875771511708160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880108.670865:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511708160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880108.670885:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880108.670889:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880108.670893:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880109.309575:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880109.309672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880109.565502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880110.333541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880110.333621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880110.589514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880111.357531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880111.357603:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880111.613499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880112.381521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880112.381611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880112.637511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880113.405589:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880113.405630:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880113.661483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880113.789608:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880113.789628:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880113.789637:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880113.789658:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5100 00000100:00080000:2.0:1788880113.789677:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880113.789690:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880113.789750:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511708288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880113.789799:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880113.789805:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880113.789808:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880113.789814:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5100 00000100:00080000:2.0:1788880113.789821:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880113.789827:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880113.789843:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880113.789861:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511708416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880113.789877:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511708288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880113.789894:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880113.789900:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880113.789903:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880113.789908:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5100 00000100:00080000:2.0:1788880113.789913:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880113.789918:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880113.789925:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880113.789942:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511708544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880113.789956:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880113.789960:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880113.789963:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880113.789968:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5100 00000100:00080000:2.0:1788880113.789976:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880113.789980:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880113.789987:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880113.789995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880113.790008:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511708672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880113.790045:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880113.790049:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880113/real 1788880113] req@ffffa061888afb80 x1875771511708288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880113.790058:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880113.790069:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880113.790097:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5100 00000100:00080000:2.0:1788880113.790114:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880113.790124:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880113.790126:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511708288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788880113.790141:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880113.790153:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880113.790163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880113.790179:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511708288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880113.790181:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511708800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880113.790243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880113.790280:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880113.790295:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880113/real 1788880113] req@ffffa061888af100 x1875771511708416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880113.790311:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511708416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880113.790323:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880113.790327:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880113.790332:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880113.790339:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880113.790360:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880113.790452:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880113/real 1788880113] req@ffffa061888ae300 x1875771511708544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880113.790480:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511708544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880113.790503:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880113.790507:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880113.790512:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880113.790521:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880113.790552:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880113.790566:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880113/real 1788880113] req@ffffa061888aea00 x1875771511708672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880113.790580:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511708672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880113.790591:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880113.790595:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880113.790598:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880113.790605:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880113.790624:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880113.790635:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880113/real 1788880113] req@ffffa061888adc00 x1875771511708800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880113.790649:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511708800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880113.790659:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880113.790662:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880113.790667:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511708800:192.168.203.111@tcp:8:kworker.0 00010000:00010000:2.0:1788880114.429580:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880114.429637:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788880114.685505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880115.453494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880115.453593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880115.709530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880116.477553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880116.477637:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880116.733548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880117.501565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880117.501679:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880117.757540:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880118.525541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880118.525634:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880118.781542:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880118.909540:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880118.909557:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880118.909564:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880118.909586:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5105 00000100:00080000:2.0:1788880118.909608:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880118.909622:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880118.909687:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511708928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880118.909736:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880118.909741:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880118.909744:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880118.909752:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5105 00000100:00080000:2.0:1788880118.909761:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880118.909770:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880118.909787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880118.909805:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511709056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880118.909819:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511708928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880118.909831:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880118.909839:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880118.909841:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880118.909848:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5105 00000100:00080000:2.0:1788880118.909853:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880118.909858:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880118.909866:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880118.909880:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511709184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880118.909892:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880118.909898:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880118.909902:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880118.909907:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5105 00000100:00080000:2.0:1788880118.909912:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880118.909917:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880118.909925:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880118.909939:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511709312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880118.909959:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880118.909968:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880118.909974:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880118.909978:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880118.909984:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5105 00000100:00080000:2.0:1788880118.909990:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880118.909996:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880118.910002:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880118.910021:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511709440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880118.910055:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880118/real 1788880118] req@ffffa061888aed80 x1875771511708928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880118.910088:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511708928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880118.910103:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880118.910110:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880118.910115:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511708928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880118.910152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880118.910187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880118.910204:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880118/real 1788880118] req@ffffa061888ac700 x1875771511709056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880118.910220:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511709056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880118.910232:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880118.910236:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880118.910241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880118.910249:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880118.910269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880118.910281:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880118/real 1788880118] req@ffffa061888ac380 x1875771511709184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880118.910296:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511709184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880118.910305:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880118.910309:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880118.910312:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880118.910320:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880118.910376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880118.910427:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880118/real 1788880118] req@ffffa061888ad880 x1875771511709312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880118.910453:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511709312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880118.910470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880118.910476:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880118.910483:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880118.910498:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880118.910538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880118.910555:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880118/real 1788880118] req@ffffa061888ae680 x1875771511709440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880118.910571:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511709440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880118.910582:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880118.910586:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880118.910590:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880119.549491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880119.549648:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880119.805526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880120.573542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880120.573605:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880120.829480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880121.597576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880121.597662:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880121.853532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880122.621535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880122.621650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880122.877529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880123.645540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880123.645692:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880123.901504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880124.029530:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880124.029547:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880124.029553:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880124.029569:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5110 00000100:00080000:3.0:1788880124.029587:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880124.029599:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880124.029651:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511709568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880124.029692:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880124.029697:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880124.029700:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880124.029707:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5110 00000100:00080000:3.0:1788880124.029713:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880124.029719:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880124.029734:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880124.029761:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511709696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880124.029762:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511709568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880124.029802:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880124.029807:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880124.029810:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880124.029815:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5110 00000100:00080000:3.0:1788880124.029821:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880124.029825:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880124.029832:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880124.029847:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511709824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880124.029860:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880124.029866:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880124.029872:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880124.029874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880124.029882:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5110 00000100:00080000:3.0:1788880124.029889:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880124.029896:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880124.029903:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880124.029917:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511709952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880124.029930:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880124.029936:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788880124.029943:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880124/real 1788880124] req@ffffa06188872d80 x1875771511709568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880124.029943:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880124.029956:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5110 00000100:00080000:3.0:1788880124.029965:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880124.029977:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880124.029989:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880124.030008:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511710080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880124.030020:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511709568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880124.030050:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880124.030056:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880124.030063:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511709568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880124.030097:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880124.030132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880124.030150:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880124/real 1788880124] req@ffffa06188873b80 x1875771511709696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880124.030166:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511709696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880124.030178:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880124.030182:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880124.030186:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880124.030195:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880124.030215:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880124.030227:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880124/real 1788880124] req@ffffa06188872300 x1875771511709824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880124.030241:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511709824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880124.030251:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880124.030254:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880124.030258:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880124.030265:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880124.030285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880124.030329:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880124/real 1788880124] req@ffffa06188872a00 x1875771511709952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880124.030345:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511709952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880124.030356:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880124.030360:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880124.030364:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511709952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880124.030372:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880124.030432:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880124.030447:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880124/real 1788880124] req@ffffa06188873480 x1875771511710080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880124.030463:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511710080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880124.030473:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880124.030477:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880124.030480:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880124.669484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880124.669512:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880124.925444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880125.693484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880125.693660:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880125.949496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880126.717478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880126.717653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880126.973480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880127.741523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880127.741629:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880127.997495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880128.765498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880128.765611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880129.021510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880129.149519:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880129.149538:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880129.149545:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880129.149560:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5115 00000100:00080000:3.0:1788880129.149577:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880129.149588:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880129.149637:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511710208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880129.149679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880129.149685:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880129.149688:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880129.149694:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5115 00000100:00080000:3.0:1788880129.149700:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880129.149706:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880129.149719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880129.149732:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511710208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880129.149739:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511710336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880129.149758:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880129.149766:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880129.149769:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880129.149774:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5115 00000100:00080000:3.0:1788880129.149778:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880129.149783:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880129.149790:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880129.149804:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511710464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880129.149817:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880129.149821:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880129.149823:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880129.149830:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5115 00000800:00000100:0.0:1788880129.149839:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880129.149840:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880129.149853:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880129.149860:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880129.149875:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511710592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880129.149888:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880129.149893:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880129.149896:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880129.149900:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880129/real 1788880129] req@ffffa06188870700 x1875771511710208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880129.149906:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5115 00000100:00080000:3.0:1788880129.149915:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880129.149926:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880129.149940:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880129.149954:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511710208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880129.149960:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511710720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880129.150002:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880129.150012:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880129.150019:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511710208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880129.150051:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880129.150079:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880129.150094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880129/real 1788880129] req@ffffa06188872680 x1875771511710336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880129.150110:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511710336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880129.150129:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880129.150134:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880129.150138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880129.150145:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880129.150164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880129.150176:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880129/real 1788880129] req@ffffa06188873100 x1875771511710464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880129.150190:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511710464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880129.150200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880129.150204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880129.150208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880129.150215:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880129.150234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880129.150245:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880129/real 1788880129] req@ffffa06188871500 x1875771511710592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880129.150295:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511710592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880129.150306:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880129.150310:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880129.150314:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880129.150322:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880129.150344:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880129.150357:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880129/real 1788880129] req@ffffa06188873480 x1875771511710720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880129.150371:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511710720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880129.150412:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880129.150416:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880129.150420:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880129.789510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880129.789625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880130.045495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880130.813559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880130.813705:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880131.069484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880131.837496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880131.837603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880132.093535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880132.861552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880132.861693:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880133.117485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880133.885467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880133.885576:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880134.141484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880134.269526:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880134.269541:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880134.269548:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880134.269566:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5120 00000100:00080000:0.0:1788880134.269582:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880134.269592:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880134.269640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511710848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880134.269681:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880134.269686:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880134.269689:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880134.269694:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5120 00000100:00080000:0.0:1788880134.269699:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880134.269705:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880134.269719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880134.269736:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511710976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880134.269751:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880134.269756:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880134.269759:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880134.269768:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511710848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880134.269769:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5120 00000100:00080000:0.0:1788880134.269782:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880134.269789:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880134.269795:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880134.269806:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511711104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880134.269820:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880134.269824:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880134.269827:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880134.269832:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5120 00000100:00080000:0.0:1788880134.269837:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880134.269841:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880134.269847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880134.269863:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511711232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880134.269875:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880134.269881:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880134.269885:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788880134.269889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788880134.269898:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5120 00000100:00080000:0.0:1788880134.269911:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880134.269920:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880134.269931:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880134.269948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880134/real 1788880134] req@ffffa06198aff800 x1875771511710848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880134.269961:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511711360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880134.269992:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511710848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880134.270010:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880134.270017:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880134.270026:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511710848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880134.270069:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880134.270109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880134.270127:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880134/real 1788880134] req@ffffa06198affb80 x1875771511710976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880134.270143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511710976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880134.270155:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880134.270160:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880134.270164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511710976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880134.270171:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880134.270191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880134.270202:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880134/real 1788880134] req@ffffa06198afd880 x1875771511711104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880134.270216:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511711104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880134.270250:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880134.270254:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880134.270258:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880134.270266:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880134.270290:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880134.270303:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880134/real 1788880134] req@ffffa06198aff480 x1875771511711232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880134.270318:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511711232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880134.270329:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880134.270332:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880134.270336:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880134.270342:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880134.270361:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880134.270372:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880134/real 1788880134] req@ffffa06198afed80 x1875771511711360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880134.270433:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511711360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880134.270444:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880134.270448:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880134.270452:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880134.909518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880134.909631:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880135.165479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880135.933537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880135.933623:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880136.189454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880136.957457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880136.957518:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880137.213528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880137.981508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880137.981621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880138.237462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880139.005485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880139.005583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880139.261477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880139.389484:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880139.389493:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880139.389497:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880139.389515:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5126 00000100:00080000:1.0:1788880139.389526:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880139.389533:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880139.389569:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511711488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880139.389593:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880139.389598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880139.389599:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880139.389603:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5126 00000100:00080000:1.0:1788880139.389605:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880139.389608:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880139.389616:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880139.389625:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511711616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880139.389633:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880139.389635:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880139.389636:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880139.389638:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5126 00000100:00080000:1.0:1788880139.389640:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880139.389643:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880139.389646:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880139.389653:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511711744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880139.389660:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880139.389663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880139.389664:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880139.389667:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5126 00000100:00080000:1.0:1788880139.389669:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880139.389674:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880139.389675:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511711488:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788880139.389678:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880139.389689:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511711872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880139.389696:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880139.389699:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880139.389700:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880139.389703:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5126 00000100:00080000:1.0:1788880139.389705:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880139.389708:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880139.389711:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880139.389719:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511712000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880139.389746:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880139.389782:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880139/real 1788880139] req@ffffa0618869b100 x1875771511711488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880139.389799:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511711488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880139.389809:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880139.389812:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880139.389816:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511711488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880139.389821:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880139.389836:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880139.389844:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880139/real 1788880139] req@ffffa06188698380 x1875771511711616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880139.389853:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511711616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880139.389859:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880139.389861:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880139.389864:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880139.389868:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880139.389879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880139.389898:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880139/real 1788880139] req@ffffa0618869b800 x1875771511711744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880139.389907:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511711744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880139.389913:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880139.389916:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880139.389918:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880139.389939:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880139.389953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880139.389960:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880139/real 1788880139] req@ffffa06188699c00 x1875771511711872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880139.389968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511711872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880139.389974:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880139.389976:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880139.389978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511711872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880139.389982:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880139.389992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880139.389999:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880139/real 1788880139] req@ffffa0618869aa00 x1875771511712000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880139.390007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511712000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880139.390018:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880139.390020:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880139.390022:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880140.029497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880140.029635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880140.285528:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880141.053524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880141.053601:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880141.309547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880142.077546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880142.077640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880142.333530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880143.101500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880143.101619:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880143.357485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880144.125500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880144.125615:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880144.381493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880144.509541:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880144.509554:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880144.509560:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880144.509576:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5131 00000100:00080000:1.0:1788880144.509592:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880144.509604:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880144.509655:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511712128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880144.509701:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880144.509708:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880144.509712:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880144.509717:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5131 00000100:00080000:1.0:1788880144.509723:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880144.509728:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880144.509743:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880144.509769:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511712256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880144.509769:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511712128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880144.509797:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880144.509802:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880144.509804:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880144.509809:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5131 00000100:00080000:1.0:1788880144.509813:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880144.509818:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880144.509825:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880144.509841:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511712384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880144.509854:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880144.509859:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880144.509861:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880144.509865:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5131 00000100:00080000:1.0:1788880144.509871:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788880144.509879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880144.509882:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880144.509892:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880144.509908:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511712512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880144.509921:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880144.509926:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880144.509928:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880144.509937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880144/real 1788880144] req@ffffa0618869aa00 x1875771511712128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880144.509940:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5131 00000100:00080000:1.0:1788880144.509950:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880144.509963:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880144.509977:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880144.509983:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511712128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880144.510001:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788880144.510007:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511712640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880144.510011:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880144.510025:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511712128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880144.510070:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880144.510104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880144.510122:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880144/real 1788880144] req@ffffa06188699c00 x1875771511712256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880144.510138:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511712256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880144.510176:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880144.510180:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880144.510185:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880144.510193:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880144.510216:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880144.510230:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880144/real 1788880144] req@ffffa0618869b800 x1875771511712384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880144.510245:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511712384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880144.510254:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880144.510257:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880144.510261:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880144.510268:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880144.510288:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880144.510299:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880144/real 1788880144] req@ffffa06188698380 x1875771511712512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880144.510313:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511712512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880144.510324:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880144.510328:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880144.510332:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880144.510338:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880144.510356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880144.510366:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880144/real 1788880144] req@ffffa0618869b100 x1875771511712640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880144.510380:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511712640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880144.510423:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880144.510427:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880144.510431:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880145.149557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880145.149685:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880145.405461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880146.173472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880146.173699:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880146.429477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880147.197492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880147.197605:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880147.453512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880148.221512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880148.221608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880148.477434:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880149.245490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880149.245618:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880149.501490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880149.629568:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880149.629580:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880149.629585:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880149.629598:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5136 00000100:00080000:3.0:1788880149.629613:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880149.629624:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880149.629670:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511712768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880149.629715:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880149.629720:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880149.629723:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880149.629728:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5136 00000100:00080000:3.0:1788880149.629733:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880149.629737:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880149.629750:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880149.629765:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511712896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880149.629778:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880149.629782:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880149.629784:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880149.629789:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5136 00000100:00080000:3.0:1788880149.629794:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880149.629800:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880149.629806:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880149.629819:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511713024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880149.629832:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880149.629840:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1788880149.629842:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511712768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880149.629844:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880149.629861:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5136 00000100:00080000:3.0:1788880149.629873:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880149.629877:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880149.629883:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880149.629899:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511713152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880149.629911:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880149.629916:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880149.629919:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880149.629923:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5136 00000100:00080000:3.0:1788880149.629941:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880149.629946:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880149.629952:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880149.629968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880149.629971:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511713280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880149.630042:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880149/real 1788880149] req@ffffa06188872a00 x1875771511712768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880149.630084:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511712768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880149.630102:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880149.630107:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880149.630138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511712768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880149.630150:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880149.630186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880149.630205:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880149/real 1788880149] req@ffffa06188872300 x1875771511712896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880149.630220:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511712896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880149.630231:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880149.630236:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880149.630240:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511712896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880149.630270:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880149.630291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880149.630303:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880149/real 1788880149] req@ffffa06188873b80 x1875771511713024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880149.630326:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511713024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880149.630336:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880149.630340:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880149.630344:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880149.630350:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880149.630370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880149.630417:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880149/real 1788880149] req@ffffa06188872d80 x1875771511713152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880149.630435:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511713152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880149.630445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880149.630449:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880149.630453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880149.630461:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880149.630484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880149.630498:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880149/real 1788880149] req@ffffa06188870a80 x1875771511713280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880149.630512:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511713280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880149.630530:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880149.630534:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880149.630538:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880150.269518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880150.269623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880150.525489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880151.293497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880151.293669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880151.549490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880152.317510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880152.317623:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880152.573508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880153.341543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880153.341720:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880153.597478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880154.365564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880154.365718:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880154.621512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880154.749519:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880154.749532:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880154.749539:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880154.749555:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5141 00000100:00080000:3.0:1788880154.749573:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880154.749588:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880154.749648:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511713408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880154.749694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880154.749700:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880154.749702:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880154.749708:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5141 00000100:00080000:3.0:1788880154.749715:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880154.749722:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880154.749754:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880154.749780:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511713408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880154.749791:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511713536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880154.749808:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880154.749813:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880154.749816:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880154.749837:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5141 00000100:00080000:3.0:1788880154.749844:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880154.749850:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880154.749859:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880154.749877:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511713664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880154.749890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1788880154.749901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880154.749903:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880154.749911:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880154.749918:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5141 00000100:00080000:3.0:1788880154.749924:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880154.749930:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880154.749938:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880154.749956:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511713792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880154.749967:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880154/real 1788880154] req@ffffa06188870380 x1875771511713408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880154.749975:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880154.749995:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880154.750005:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880154.750017:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5141 00000100:00100000:2.0:1788880154.750023:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511713408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880154.750029:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880154.750048:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:2.0:1788880154.750051:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880154.750064:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:3.0:1788880154.750064:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880154.750074:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511713408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880154.750124:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511713920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880154.750139:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880154.750179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880154.750199:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880154/real 1788880154] req@ffffa06188871880 x1875771511713536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880154.750215:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511713536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880154.750226:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880154.750230:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880154.750235:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880154.750243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880154.750263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880154.750275:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880154/real 1788880154] req@ffffa06188873800 x1875771511713664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880154.750289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511713664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880154.750298:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880154.750302:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880154.750305:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880154.750313:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880154.750332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880154.750343:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880154/real 1788880154] req@ffffa06188871c00 x1875771511713792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880154.750356:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511713792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880154.750367:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880154.750371:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880154.750375:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880154.750442:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880154.750468:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880154.750483:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880154/real 1788880154] req@ffffa06188870a80 x1875771511713920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880154.750497:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511713920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880154.750508:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880154.750512:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880154.750516:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511713920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880155.389509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880155.389640:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880155.645523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880156.413473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880156.413597:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880156.669509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880157.437533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880157.437722:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880157.693506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880158.461541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880158.461701:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880158.717503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880159.485507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880159.485606:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880159.741486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880159.869464:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880159.869473:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880159.869478:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880159.869490:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5146 00000100:00080000:3.0:1788880159.869502:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880159.869510:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880159.869568:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511714048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880159.869599:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880159.869603:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880159.869605:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880159.869609:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5146 00000100:00080000:3.0:1788880159.869612:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880159.869616:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880159.869628:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880159.869660:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511714176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880159.869673:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880159.869678:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880159.869681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880159.869686:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5146 00000100:00080000:3.0:1788880159.869692:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880159.869697:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880159.869702:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880159.869713:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511714304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880159.869736:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788880159.869742:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511714048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880159.869742:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880159.869747:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880159.869759:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5146 00000100:00080000:3.0:1788880159.869769:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880159.869774:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880159.869780:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880159.869798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511714432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880159.869822:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880159.869824:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880159.869825:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880159.869828:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5146 00000100:00080000:3.0:1788880159.869831:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880159.869834:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880159.869837:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880159.869845:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511714560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880159.869864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880159.869932:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880159/real 1788880159] req@ffffa06188872d80 x1875771511714048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880159.869957:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511714048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880159.869972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880159.869978:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880159.869984:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511714048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880159.870009:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880159.870031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880159.870078:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880159/real 1788880159] req@ffffa06188873b80 x1875771511714176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880159.870087:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511714176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880159.870094:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880159.870096:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880159.870098:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880159.870120:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880159.870134:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880159.870141:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880159/real 1788880159] req@ffffa06188872300 x1875771511714304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880159.870154:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511714304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880159.870159:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880159.870161:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880159.870163:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880159.870166:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880159.870178:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880159.870184:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880159/real 1788880159] req@ffffa06188872a00 x1875771511714432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880159.870191:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511714432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880159.870196:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880159.870198:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880159.870200:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880159.870203:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880159.870212:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880159.870218:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880159/real 1788880159] req@ffffa06188873480 x1875771511714560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880159.870225:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511714560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880159.870235:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880159.870237:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880159.870239:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880160.509547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880160.509669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880160.765567:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880161.533497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880161.533637:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880161.789494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880162.557481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880162.557670:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880162.813493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880163.581511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880163.581603:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880163.837504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880164.605498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880164.605675:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880164.861449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880164.989503:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880164.989515:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880164.989521:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880164.989536:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5151 00000100:00080000:3.0:1788880164.989554:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880164.989564:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880164.989614:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511714688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880164.989650:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880164.989655:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880164.989658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880164.989666:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5151 00000100:00080000:3.0:1788880164.989671:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880164.989677:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880164.989691:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880164.989708:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511714816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880164.989727:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788880164.989733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511714688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880164.989735:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880164.989742:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880164.989754:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5151 00000100:00080000:3.0:1788880164.989758:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880164.989762:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880164.989769:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880164.989785:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511714944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880164.989798:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880164.989804:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880164.989806:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880164.989811:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5151 00000100:00080000:3.0:1788880164.989828:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880164.989833:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880164.989841:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880164.989846:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880164.989864:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511715072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880164.989878:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880164.989883:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880164.989886:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880164.989892:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5151 00000100:00080000:3.0:1788880164.989898:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788880164.989907:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880164/real 1788880164] req@ffffa06188871500 x1875771511714688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880164.989908:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880164.989932:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880164.989941:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511714688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880164.989956:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511715200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880164.989962:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880164.989979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880164.989989:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511714688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880164.990062:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880164.990097:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880164.990116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880164/real 1788880164] req@ffffa06188873100 x1875771511714816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880164.990132:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511714816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880164.990144:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880164.990148:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880164.990152:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880164.990160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880164.990180:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880164.990192:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880164/real 1788880164] req@ffffa06188872680 x1875771511714944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880164.990206:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511714944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880164.990216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880164.990219:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880164.990223:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511714944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880164.990230:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880164.990250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880164.990260:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880164/real 1788880164] req@ffffa06188870700 x1875771511715072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880164.990274:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511715072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880164.990285:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880164.990288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880164.990292:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880164.990299:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880164.990317:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880164.990328:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880164/real 1788880164] req@ffffa06188873480 x1875771511715200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880164.990340:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511715200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880164.990350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880164.990353:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880164.990357:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880165.629512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880165.629596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880165.885485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880166.653525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880166.653608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880166.909494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880167.677508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880167.677683:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880167.933477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880168.701515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880168.701643:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880168.957513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880169.725497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880169.725649:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880169.981484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880170.109543:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880170.109554:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880170.109560:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880170.109572:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5156 00000100:00080000:3.0:1788880170.109585:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880170.109594:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880170.109643:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511715328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880170.109672:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880170.109676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880170.109678:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880170.109682:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5156 00000100:00080000:3.0:1788880170.109686:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880170.109689:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880170.109700:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880170.109716:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511715328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880170.109716:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511715456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880170.109746:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880170.109748:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880170.109751:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880170.109756:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5156 00000100:00080000:3.0:1788880170.109759:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880170.109764:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880170.109768:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880170.109779:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511715584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880170.109787:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880170.109790:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880170.109792:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880170.109796:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5156 00000100:00080000:3.0:1788880170.109802:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880170.109807:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880170.109813:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880170.109831:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511715712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880170.109845:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715456:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788880170.109848:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880170.109855:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880170.109859:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880170.109867:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5156 00000100:00080000:3.0:1788880170.109876:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788880170.109877:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715584:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788880170.109891:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880170.109899:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880170.109909:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880170.109936:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511715840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880170.109990:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880170.749535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880170.749688:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880171.005503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880171.773535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880171.773681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880172.029499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880172.797543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880172.797644:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880173.053495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880173.181689:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880173.181745:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880173.181767:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880173.181814:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880173.181825:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880173.181835:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880173.181843:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788880173.181976:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880170/real 1788880173] req@ffffa06188870a80 x1875771511715840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880173.182027:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511715840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880173.182049:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880173.182057:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880173.182066:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880173.182080:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880170/real 1788880173] req@ffffa06188872d80 x1875771511715712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880173.182111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511715712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880173.182125:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880173.182132:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880173.182138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880173.182161:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880170/real 1788880173] req@ffffa06188872300 x1875771511715456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880173.182187:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511715456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880173.182204:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880173.182211:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880173.182217:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880173.182231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880170/real 1788880173] req@ffffa06188873b80 x1875771511715584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880173.182256:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511715584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880173.182272:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880173.182279:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880173.182285:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511715584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880173.182297:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880170/real 1788880173] req@ffffa06188872a00 x1875771511715328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880173.182323:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511715328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880173.182338:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880173.182345:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880173.182352:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511715328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880173.821510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880173.821680:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880174.077524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880174.845546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880174.845632:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880175.101521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880175.229536:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880175.229549:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880175.229574:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880175.229592:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5161 00000100:00080000:3.0:1788880175.229610:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880175.229624:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880175.229699:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511715968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880175.229738:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880175.229747:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880175.229750:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880175.229756:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5161 00000100:00080000:3.0:1788880175.229761:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880175.229766:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880175.229780:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880175.229803:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511716096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880175.229824:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880175.229832:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880175.229852:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880175.229861:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5161 00000100:00080000:3.0:1788880175.229868:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880175.229876:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880175.229891:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511715968:192.168.203.111@tcp:250:kworker.0 00000008:00100000:3.0:1788880175.229891:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880175.230002:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511716224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880175.230028:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880175.230053:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880175.230058:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880175.230065:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5161 00000100:00080000:3.0:1788880175.230079:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880175.230090:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880175.230100:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880175.230110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880175.230137:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511716352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880175.230160:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880175.230166:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880175.230170:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880175.230179:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5161 00000100:00080000:3.0:1788880175.230187:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880175.230205:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880175.230207:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880175/real 1788880175] req@ffffa06188871c00 x1875771511715968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788880175.230219:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880175.230260:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511716480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880175.230268:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511715968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880175.230323:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880175.230332:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880175.230340:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511715968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880175.230355:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880175.230459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880175.230486:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880175/real 1788880175] req@ffffa06188873800 x1875771511716096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880175.230508:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511716096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880175.230523:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880175.230529:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880175.230535:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880175.230577:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880175.230621:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880175.230647:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880175/real 1788880175] req@ffffa06188871880 x1875771511716224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880175.230671:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511716224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880175.230687:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880175.230694:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880175.230700:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880175.230718:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880175.230756:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880175.230777:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880175/real 1788880175] req@ffffa06188870380 x1875771511716352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880175.230807:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511716352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880175.230822:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880175.230828:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880175.230833:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880175.230845:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880175.230878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880175.230903:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880175/real 1788880175] req@ffffa06188872a00 x1875771511716480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880175.230958:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511716480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880175.230974:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880175.230979:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880175.230985:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880175.869556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880175.869656:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880176.125476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880176.893528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880176.893637:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880177.149510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880177.917473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880177.917667:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880178.173567:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880178.941525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880178.941600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880179.197493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880179.965482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880179.965620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880180.221490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880180.349581:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880180.349591:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880180.349597:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880180.349611:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5167 00000100:00080000:0.0:1788880180.349626:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880180.349635:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880180.349684:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511716608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880180.349723:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880180.349729:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880180.349732:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880180.349737:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5167 00000100:00080000:0.0:1788880180.349742:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880180.349747:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880180.349760:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880180.349774:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511716736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880180.349788:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880180.349792:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880180.349794:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880180.349798:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5167 00000100:00080000:0.0:1788880180.349802:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880180.349806:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880180.349812:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880180.349825:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511716864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880180.349839:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880180.349844:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880180.349847:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880180.349855:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511716608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880180.349858:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5167 00000100:00080000:0.0:1788880180.349871:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880180.349878:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880180.349886:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880180.349928:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511716992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880180.349940:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880180.349946:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880180.349949:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880180.349953:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5167 00000100:00080000:0.0:1788880180.349958:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880180.349963:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880180.349969:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880180.349984:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511717120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880180.350013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880180.350070:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880180/real 1788880180] req@ffffa06198afed80 x1875771511716608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880180.350097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511716608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880180.350111:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880180.350117:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880180.350122:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511716608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880180.350133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880180.350165:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880180.350181:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880180/real 1788880180] req@ffffa06198aff480 x1875771511716736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880180.350196:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511716736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880180.350206:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880180.350210:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880180.350214:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880180.350245:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880180.350266:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880180.350278:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880180/real 1788880180] req@ffffa06198afd880 x1875771511716864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880180.350292:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511716864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880180.350302:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880180.350306:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880180.350309:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880180.350316:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880180.350335:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880180.350346:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880180/real 1788880180] req@ffffa06198affb80 x1875771511716992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880180.350369:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511716992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880180.350379:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880180.350425:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880180.350431:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511716992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880180.350439:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880180.350464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880180.350478:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880180/real 1788880180] req@ffffa06198aff800 x1875771511717120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880180.350492:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511717120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880180.350503:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880180.350506:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880180.350510:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880180.989517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880180.989647:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880181.245579:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880182.013516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880182.013661:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880182.269514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880183.037482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880183.037630:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880183.293520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880184.061514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880184.061681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880184.317483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880185.085538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880185.085672:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880185.341569:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880185.469613:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880185.469626:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880185.469631:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880185.469644:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5172 00000100:00080000:1.0:1788880185.469659:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880185.469669:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880185.469717:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511717248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880185.469755:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880185.469760:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880185.469763:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880185.469769:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5172 00000100:00080000:1.0:1788880185.469773:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880185.469777:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880185.469791:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880185.469805:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511717376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880185.469818:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880185.469822:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880185.469825:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880185.469830:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5172 00000100:00080000:1.0:1788880185.469833:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880185.469839:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880185.469844:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880185.469882:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511717504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880185.469897:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788880185.469904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511717248:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880185.469905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880185.469912:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880185.469927:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5172 00000100:00080000:1.0:1788880185.469932:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880185.469936:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880185.469944:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880185.469961:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511717632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880185.469973:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880185.469979:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880185.469981:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880185.469985:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5172 00000100:00080000:1.0:1788880185.469991:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880185.469995:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880185.470002:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880185.470024:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511717760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880185.470027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880185.470098:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880185/real 1788880185] req@ffffa06188699f80 x1875771511717248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880185.470130:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511717248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880185.470146:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880185.470152:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880185.470157:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511717248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880185.470167:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880185.470201:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880185.470217:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880185/real 1788880185] req@ffffa06188698000 x1875771511717376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880185.470232:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511717376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880185.470243:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880185.470247:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880185.470251:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880185.470282:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880185.470302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880185.470314:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880185/real 1788880185] req@ffffa0618869a680 x1875771511717504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880185.470337:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511717504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880185.470347:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880185.470350:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880185.470354:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880185.470361:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880185.470422:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880185.470438:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880185/real 1788880185] req@ffffa0618869bb80 x1875771511717632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880185.470454:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511717632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880185.470464:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880185.470468:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880185.470472:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880185.470479:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880185.470502:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880185.470514:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880185/real 1788880185] req@ffffa0618869ad80 x1875771511717760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880185.470528:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511717760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880185.470547:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880185.470551:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880185.470555:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511717760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880186.109531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880186.109719:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880186.365537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880187.133497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880187.133655:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880187.389526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880188.157542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880188.157732:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880188.413603:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880189.181493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880189.181643:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880189.437580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880190.205533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880190.205629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880190.461557:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880190.589544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880190.589560:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880190.589568:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880190.589585:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5177 00000100:00080000:1.0:1788880190.589604:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880190.589616:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880190.589677:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511717888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880190.589718:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880190.589724:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880190.589727:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880190.589734:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5177 00000100:00080000:1.0:1788880190.589739:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880190.589745:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880190.589760:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880190.589777:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511718016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880190.589792:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880190.589797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880190.589800:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880190.589806:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5177 00000100:00080000:1.0:1788880190.589811:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880190.589817:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880190.589853:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880190.589871:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511718144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880190.589885:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880190.589891:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880190.589893:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880190.589899:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5177 00000100:00080000:1.0:1788880190.589905:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788880190.589918:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511717888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880190.589920:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880190.589945:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880190.589965:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511718272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880190.589981:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880190.589986:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880190.589989:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880190.589994:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5177 00000100:00080000:1.0:1788880190.590000:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880190.590005:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880190.590012:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880190.590026:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511718400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880190.590060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880190.590133:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880190/real 1788880190] req@ffffa0618869ad80 x1875771511717888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880190.590162:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511717888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880190.590179:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880190.590184:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880190.590190:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511717888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880190.590201:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880190.590236:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880190.590252:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880190/real 1788880190] req@ffffa0618869bb80 x1875771511718016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880190.590269:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511718016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880190.590282:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880190.590288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880190.590295:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880190.590308:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880190.590346:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880190.590370:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880190/real 1788880190] req@ffffa0618869a680 x1875771511718144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880190.590471:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511718144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880190.590482:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880190.590486:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880190.590491:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880190.590526:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880190.590559:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880190.590574:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880190/real 1788880190] req@ffffa06188698000 x1875771511718272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880190.590589:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511718272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880190.590608:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880190.590612:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880190.590616:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880190.590623:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880190.590642:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880190.590654:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880190/real 1788880190] req@ffffa06188699f80 x1875771511718400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880190.590667:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511718400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880190.590677:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880190.590681:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880190.590684:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880191.229501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880191.229688:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880191.485611:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880192.253484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880192.253610:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880192.509487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880193.277515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880193.277598:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880193.533552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880194.301528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880194.301618:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880194.557507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880195.325515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880195.325658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880195.581487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880195.709641:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880195.709652:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880195.709658:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880195.709671:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5182 00000100:00080000:3.0:1788880195.709686:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880195.709696:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880195.709743:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511718528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880195.709776:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880195.709782:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880195.709784:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880195.709817:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5182 00000100:00080000:3.0:1788880195.709824:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880195.709831:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880195.709846:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880195.709867:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511718656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880195.709875:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511718528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880195.709899:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880195.709905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880195.709908:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880195.709914:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5182 00000100:00080000:3.0:1788880195.709919:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880195.709923:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880195.709929:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880195.709944:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511718784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880195.709957:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880195.709962:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880195.709964:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880195.709968:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5182 00000100:00080000:3.0:1788880195.709983:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880195.709988:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880195.709995:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880195.710002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880195.710018:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511718912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880195.710041:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880195.710046:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880195.710049:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880195.710058:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880195/real 1788880195] req@ffffa06188873b80 x1875771511718528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880195.710062:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5182 00000100:00080000:3.0:1788880195.710074:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880195.710087:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880195.710098:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880195.710110:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511718528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880195.710118:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511719040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880195.710153:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880195.710163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880195.710170:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511718528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880195.710221:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880195.710251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880195.710267:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880195/real 1788880195] req@ffffa06188872300 x1875771511718656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880195.710284:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511718656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880195.710296:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880195.710300:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880195.710304:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880195.710311:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880195.710331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880195.710343:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880195/real 1788880195] req@ffffa06188872d80 x1875771511718784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880195.710357:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511718784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880195.710376:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880195.710415:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880195.710422:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880195.710430:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880195.710457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880195.710472:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880195/real 1788880195] req@ffffa06188870a80 x1875771511718912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880195.710487:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511718912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880195.710497:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880195.710501:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880195.710505:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511718912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880195.710512:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880195.710532:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880195.710543:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880195/real 1788880195] req@ffffa06188873480 x1875771511719040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880195.710564:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511719040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880195.710582:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880195.710587:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880195.710595:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880196.349539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880196.349572:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880196.605452:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880197.373472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880197.373543:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880197.629510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880198.397508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880198.397672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880198.653484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880199.421539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880199.421650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880199.677474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880200.445549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880200.445632:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880200.701472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880200.829582:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880200.829597:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880200.829604:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880200.829615:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5187 00000100:00080000:1.0:1788880200.829629:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880200.829637:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880200.829686:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511719168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880200.829731:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880200.829737:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880200.829741:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880200.829749:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5187 00000100:00080000:1.0:1788880200.829772:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880200.829779:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880200.829791:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880200.829809:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511719296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880200.829823:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880200.829829:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880200.829831:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880200.829836:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5187 00000100:00080000:1.0:1788880200.829840:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788880200.829848:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511719168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880200.829862:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880200.829873:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880200.829896:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511719424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880200.829916:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880200.829924:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880200.829927:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880200.829936:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5187 00000800:00000100:2.0:1788880200.829942:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880200.829951:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880200.829961:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880200.829970:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880200.829989:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511719552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880200.829994:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880200/real 1788880200] req@ffffa0618869b100 x1875771511719168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880200.830026:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880200.830032:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880200.830035:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880200.830041:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511719168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880200.830046:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5187 00000100:00080000:1.0:1788880200.830055:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880200.830065:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:2.0:1788880200.830069:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880200.830076:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880200.830080:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511719168:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788880200.830091:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880200.830098:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788880200.830114:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511719680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880200.830131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880200.830148:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880200/real 1788880200] req@ffffa06188698380 x1875771511719296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880200.830167:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511719296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880200.830183:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880200.830189:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880200.830195:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880200.830232:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880200.830269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880200.830287:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880200/real 1788880200] req@ffffa0618869b800 x1875771511719424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880200.830301:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511719424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880200.830311:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880200.830315:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880200.830318:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880200.830325:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880200.830347:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880200.830357:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880200/real 1788880200] req@ffffa06188699c00 x1875771511719552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880200.830370:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511719552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880200.830378:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880200.830418:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880200.830423:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880200.830430:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880200.830452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880200.830464:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880200/real 1788880200] req@ffffa0618869aa00 x1875771511719680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880200.830477:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511719680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880200.830486:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880200.830489:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880200.830492:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880201.469520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880201.469624:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880201.725517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880202.493519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880202.493650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880202.749496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880203.517516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880203.517664:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880203.773522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880204.541479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880204.541669:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880204.797512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880205.565471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880205.565613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880205.821513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880205.949658:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880205.949670:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880205.949676:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880205.949692:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5192 00000100:00080000:0.0:1788880205.949707:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880205.949740:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880205.949791:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511719808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880205.949825:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880205.949831:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880205.949835:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880205.949842:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5192 00000100:00080000:0.0:1788880205.949847:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880205.949853:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880205.949867:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880205.949884:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511719936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880205.949893:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511719808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880205.949918:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880205.949924:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880205.949926:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880205.949932:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5192 00000100:00080000:0.0:1788880205.949936:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880205.949940:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880205.949947:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880205.949963:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511720064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880205.949975:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880205.949991:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880205.949994:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880205.950000:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5192 00000100:00080000:0.0:1788880205.950005:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788880205.950013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788880205.950015:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880205.950028:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880205.950045:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511720192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880205.950059:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880205.950064:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880205.950066:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880205.950074:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5192 00000100:00100000:2.0:1788880205.950075:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880205/real 1788880205] req@ffffa06198afdf80 x1875771511719808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880205.950079:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880205.950090:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880205.950104:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880205.950131:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511719808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880205.950132:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511720320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880205.950211:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880205.950218:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880205.950224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511719808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880205.950265:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880205.950306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880205.950331:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880205/real 1788880205] req@ffffa06198afdc00 x1875771511719936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880205.950357:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511719936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880205.950376:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880205.950435:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880205.950441:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511719936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880205.950450:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880205.950483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880205.950500:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880205/real 1788880205] req@ffffa06198afc700 x1875771511720064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880205.950515:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511720064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880205.950526:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880205.950529:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880205.950533:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880205.950541:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880205.950562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880205.950573:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880205/real 1788880205] req@ffffa06198afc000 x1875771511720192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880205.950587:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511720192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880205.950597:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880205.950600:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880205.950604:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880205.950610:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880205.950628:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880205.950638:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880205/real 1788880205] req@ffffa06198afea00 x1875771511720320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880205.950652:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511720320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880205.950661:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880205.950665:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880205.950668:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880206.589511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880206.589628:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880206.845460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880207.613532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880207.613600:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880207.869514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880208.637518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880208.637620:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880208.893460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880209.661514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880209.661609:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880209.917507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880210.685483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880210.685627:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880210.941507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880211.069551:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880211.069568:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880211.069575:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880211.069594:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5197 00000100:00080000:1.0:1788880211.069610:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880211.069621:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880211.069705:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511720448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880211.069756:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880211.069762:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880211.069765:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880211.069771:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5197 00000100:00080000:1.0:1788880211.069777:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880211.069786:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880211.069799:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880211.069822:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511720576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880211.069837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880211.069843:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880211.069847:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880211.069852:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511720448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880211.069855:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5197 00000100:00080000:1.0:1788880211.069868:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880211.069875:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880211.069881:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880211.069901:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511720704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880211.069913:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880211.069919:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880211.069921:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880211.069927:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5197 00000100:00080000:1.0:1788880211.069934:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880211.069942:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880211.069951:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880211.069965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880211.069977:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511720832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880211.070000:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880211.070007:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880211.070010:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880211.070020:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5197 00000100:00080000:1.0:1788880211.070029:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880211.070037:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880211.070048:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880211.070057:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880211/real 1788880211] req@ffffa0618869aa00 x1875771511720448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880211.070078:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511720960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880211.070112:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511720448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880211.070136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880211.070144:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880211.070152:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511720448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880211.070191:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880211.070238:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880211.070263:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880211/real 1788880211] req@ffffa06188699c00 x1875771511720576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880211.070281:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511720576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880211.070293:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880211.070297:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880211.070301:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880211.070309:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880211.070337:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880211.070350:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880211/real 1788880211] req@ffffa0618869b800 x1875771511720704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880211.070364:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511720704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880211.070374:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880211.070377:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880211.070424:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880211.070433:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880211.070460:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880211.070474:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880211/real 1788880211] req@ffffa06188698380 x1875771511720832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880211.070489:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511720832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880211.070499:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880211.070503:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880211.070507:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880211.070513:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880211.070532:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880211.070544:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880211/real 1788880211] req@ffffa0618869b100 x1875771511720960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880211.070566:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511720960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880211.070576:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880211.070579:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880211.070583:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511720960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880211.709500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880211.709703:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880211.965510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880212.733501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880212.733657:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880212.989482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880213.757479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880213.757652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880214.013478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880214.781519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880214.781602:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880215.037500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880215.805464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880215.805530:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880216.061471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880216.189475:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880216.189483:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880216.189487:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880216.189497:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5202 00000100:00080000:1.0:1788880216.189507:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880216.189513:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880216.189548:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511721088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880216.189570:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880216.189572:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880216.189574:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880216.189578:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5202 00000100:00080000:1.0:1788880216.189582:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880216.189585:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880216.189594:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880216.189604:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511721216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880216.189611:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880216.189614:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880216.189615:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880216.189618:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5202 00000100:00080000:1.0:1788880216.189620:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880216.189622:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880216.189625:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880216.189630:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511721088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880216.189636:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511721344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880216.189658:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880216.189661:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880216.189662:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880216.189665:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5202 00000100:00080000:1.0:1788880216.189669:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880216.189671:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880216.189675:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880216.189685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511721472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880216.189696:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880216.189698:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880216.189700:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880216.189702:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5202 00000100:00080000:1.0:1788880216.189704:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880216.189706:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880216.189709:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880216.189715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880216.189719:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511721600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880216.189756:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880216/real 1788880216] req@ffffa06188699f80 x1875771511721088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880216.189772:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511721088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880216.189780:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880216.189783:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880216.189786:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511721088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880216.189794:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880216.189812:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880216.189820:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880216/real 1788880216] req@ffffa06188698000 x1875771511721216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880216.189828:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511721216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880216.189834:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880216.189836:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880216.189837:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880216.189855:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880216.189866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880216.189872:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880216/real 1788880216] req@ffffa0618869a680 x1875771511721344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880216.189880:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511721344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880216.189885:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880216.189887:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880216.189888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880216.189892:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880216.189903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880216.189909:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880216/real 1788880216] req@ffffa0618869bb80 x1875771511721472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880216.189917:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511721472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880216.189922:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880216.189924:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880216.189926:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880216.189929:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880216.189939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880216.189944:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880216/real 1788880216] req@ffffa0618869ad80 x1875771511721600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880216.189951:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511721600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880216.189957:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880216.189958:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880216.189960:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880216.829455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880216.829548:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880217.085495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880217.853458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880217.853570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880218.109460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880218.877479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880218.877676:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880219.133498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880219.901520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880219.901695:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880220.157506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880220.925466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880220.925582:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880221.181469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880221.309672:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880221.309687:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880221.309693:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880221.309708:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5208 00000100:00080000:0.0:1788880221.309730:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880221.309742:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880221.309792:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511721728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880221.309821:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880221.309826:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880221.309828:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880221.309833:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5208 00000100:00080000:0.0:1788880221.309837:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880221.309841:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880221.309855:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880221.309875:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511721856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880221.309888:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880221.309893:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880221.309895:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880221.309906:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5208 00000100:00100000:2.0:1788880221.309907:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511721728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880221.309913:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880221.309925:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880221.309933:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880221.309953:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511721984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880221.309962:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880221.309965:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880221.309968:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880221.309972:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5208 00000100:00080000:0.0:1788880221.309974:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880221.309977:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880221.309982:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880221.309993:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511722112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880221.310003:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880221.310005:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880221.310006:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880221.310010:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5208 00000100:00080000:0.0:1788880221.310013:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880221.310020:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788880221.310025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:0.0:1788880221.310033:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880221.310049:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511722240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880221.310097:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880221/real 1788880221] req@ffffa06198afe300 x1875771511721728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880221.310124:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511721728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880221.310139:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880221.310145:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880221.310150:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511721728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880221.310185:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880221.310231:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880221.310256:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880221/real 1788880221] req@ffffa06198afc380 x1875771511721856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880221.310282:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511721856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880221.310299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880221.310306:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880221.310313:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880221.310327:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880221.310371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880221.310441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880221/real 1788880221] req@ffffa06198afca80 x1875771511721984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880221.310467:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511721984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880221.310477:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880221.310491:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880221.310495:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511721984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880221.310504:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880221.310534:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880221.310547:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880221/real 1788880221] req@ffffa06198afd500 x1875771511722112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880221.310561:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511722112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880221.310571:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880221.310575:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880221.310579:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880221.310587:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880221.310637:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880221.310655:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880221/real 1788880221] req@ffffa06198afd180 x1875771511722240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880221.310670:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511722240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880221.310689:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880221.310693:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880221.310697:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880221.949444:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880221.949531:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880222.205519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880222.973530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880222.973684:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880223.229467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880223.997506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880223.997621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880224.253460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880225.021506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880225.021634:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880225.277496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880226.045526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880226.045734:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880226.301547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880226.429646:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880226.429662:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880226.429670:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880226.429697:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5213 00000100:00080000:3.0:1788880226.429714:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880226.429727:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880226.429788:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511722368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880226.429826:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880226.429831:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880226.429834:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880226.429840:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5213 00000100:00080000:3.0:1788880226.429844:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880226.429849:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880226.429864:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880226.429879:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511722496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880226.429892:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880226.429896:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880226.429898:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880226.429902:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5213 00000100:00080000:3.0:1788880226.429907:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880226.429912:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880226.429917:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880226.429931:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511722624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880226.429950:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880226.429955:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880226.429957:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880226.429963:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5213 00000100:00080000:3.0:1788880226.429974:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788880226.429976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511722368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880226.429981:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880226.430000:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880226.430018:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511722752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880226.430031:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880226.430035:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880226.430040:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880226.430068:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5213 00000100:00080000:3.0:1788880226.430078:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880226.430085:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880226.430094:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880226.430104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880226.430120:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511722880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880226.430185:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880226/real 1788880226] req@ffffa06188870700 x1875771511722368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880226.430213:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511722368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880226.430240:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880226.430245:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880226.430251:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511722368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880226.430261:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880226.430289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880226.430303:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880226/real 1788880226] req@ffffa06188872680 x1875771511722496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880226.430319:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511722496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880226.430330:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880226.430336:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880226.430342:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880226.430436:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880226.430492:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880226.430511:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880226/real 1788880226] req@ffffa06188873100 x1875771511722624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880226.430527:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511722624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880226.430538:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880226.430542:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880226.430547:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880226.430555:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880226.430610:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880226.430626:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880226/real 1788880226] req@ffffa06188871500 x1875771511722752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880226.430641:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511722752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880226.430660:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880226.430664:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880226.430668:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880226.430676:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880226.430698:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880226.430710:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880226/real 1788880226] req@ffffa06188873480 x1875771511722880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880226.430723:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511722880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880226.430733:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880226.430737:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880226.430740:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511722880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880227.069528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880227.069656:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880227.325539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880228.093493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880228.093633:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880228.349478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880229.117530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880229.117682:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880229.373527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880230.141598:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880230.141691:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880230.397522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880231.165485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880231.165690:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880231.421454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880231.549570:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880231.549601:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880231.549608:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880231.549623:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5218 00000100:00080000:1.0:1788880231.549638:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880231.549647:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880231.549700:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511723008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880231.549732:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880231.549736:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880231.549738:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880231.549743:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5218 00000100:00080000:1.0:1788880231.549746:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880231.549750:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880231.549762:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880231.549776:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511723136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880231.549789:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788880231.549790:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511723008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880231.549805:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880231.549811:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880231.549817:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5218 00000100:00080000:1.0:1788880231.549819:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880231.549822:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880231.549827:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880231.549840:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511723264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880231.549869:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880231.549874:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880231.549877:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880231.549883:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5218 00000100:00080000:1.0:1788880231.549891:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880231.549897:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:2.0:1788880231.549906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788880231.549907:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880231.549955:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511723392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880231.549977:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880231.549997:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880231.549999:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788880231.550003:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880231/real 1788880231] req@ffffa0618869ad80 x1875771511723008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880231.550007:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5218 00000100:00080000:1.0:1788880231.550025:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880231.550033:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880231.550039:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880231.550042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511723008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880231.550054:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511723520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880231.550055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880231.550067:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880231.550074:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511723008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880231.550101:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880231.550122:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880231.550132:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880231/real 1788880231] req@ffffa0618869bb80 x1875771511723136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880231.550143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511723136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880231.550151:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880231.550153:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880231.550156:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880231.550160:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880231.550173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880231.550180:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880231/real 1788880231] req@ffffa0618869a680 x1875771511723264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880231.550189:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511723264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880231.550202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880231.550204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880231.550207:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880231.550211:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880231.550225:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880231.550231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880231/real 1788880231] req@ffffa06188698000 x1875771511723392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880231.550240:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511723392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880231.550246:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880231.550249:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880231.550251:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880231.550271:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880231.550301:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880231.550322:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880231/real 1788880231] req@ffffa06188699f80 x1875771511723520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880231.550347:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511723520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880231.550364:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880231.550370:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880231.550379:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880232.189523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880232.189631:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880232.445490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880233.213506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880233.213671:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880233.469512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880234.237567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880234.237755:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880234.493476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880235.261535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880235.261712:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880235.517495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880236.285491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880236.285668:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880236.541441:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880236.669560:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880236.669571:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880236.669576:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880236.669587:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5223 00000100:00080000:0.0:1788880236.669601:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880236.669609:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880236.669654:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511723648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880236.669679:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880236.669682:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880236.669684:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880236.669687:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5223 00000100:00080000:0.0:1788880236.669689:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880236.669692:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880236.669701:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880236.669710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511723776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880236.669719:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880236.669723:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1788880236.669724:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511723648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880236.669725:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880236.669733:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5223 00000100:00080000:0.0:1788880236.669736:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880236.669739:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880236.669742:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880236.669752:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511723904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880236.669758:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880236.669761:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880236.669763:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880236.669766:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5223 00000100:00080000:0.0:1788880236.669768:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880236.669770:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880236.669774:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880236.669780:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511724032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880236.669786:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880236.669788:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880236.669789:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880236.669792:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5223 00000100:00080000:0.0:1788880236.669795:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880236.669797:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880236.669800:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880236.669808:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511724160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880236.669840:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880236.669873:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880236.669888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880236.669904:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880237.309469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880237.309544:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880237.565469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880238.333520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880238.333574:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880238.589529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880239.357444:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880239.357523:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880239.613459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880239.741701:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880239.741749:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880239.741766:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880239.741803:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880239.741827:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880239.741841:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880239.741852:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788880239.741853:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880236/real 1788880239] req@ffffa06198afd500 x1875771511723776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880239.741879:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511723776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880239.741890:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880239.741896:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880239.741901:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880239.741910:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880236/real 1788880239] req@ffffa06198afca80 x1875771511723904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880239.741924:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511723904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880239.741933:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880239.741937:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880239.741941:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511723904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880239.741950:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880236/real 1788880239] req@ffffa06198afc380 x1875771511724032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880239.741969:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511724032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880239.741978:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880239.741982:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880239.741985:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880239.741993:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880236/real 1788880239] req@ffffa06198afe300 x1875771511724160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880239.742006:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511724160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880239.742015:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880239.742019:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880239.742023:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880239.742030:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880236/real 1788880239] req@ffffa06198afd180 x1875771511723648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880239.742050:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511723648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880239.742058:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880239.742062:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880239.742066:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511723648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880240.381515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880240.381563:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880240.637559:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880241.405541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880241.405604:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880241.661454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880241.789555:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880241.789563:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880241.789567:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880241.789576:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5228 00000100:00080000:1.0:1788880241.789585:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880241.789593:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880241.789631:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511724288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880241.789654:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880241.789657:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880241.789658:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880241.789661:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5228 00000100:00080000:1.0:1788880241.789664:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880241.789666:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880241.789675:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880241.789684:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511724416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880241.789691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880241.789693:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880241.789694:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880241.789696:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5228 00000100:00080000:1.0:1788880241.789698:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880241.789701:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880241.789704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880241.789712:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511724544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880241.789721:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880241.789726:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880241.789728:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880241.789732:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5228 00000100:00080000:1.0:1788880241.789736:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880241.789740:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880241.789746:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880241.789759:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511724672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880241.789774:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788880241.789780:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511724288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880241.789785:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880241.789795:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880241.789806:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5228 00000100:00080000:1.0:1788880241.789810:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880241.789814:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880241.789821:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880241.789835:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511724800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880241.789922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880241.789999:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880241/real 1788880241] req@ffffa0618869b100 x1875771511724288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880241.790036:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511724288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880241.790057:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880241.790067:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880241.790075:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511724288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880241.790089:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880241.790136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880241.790164:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880241/real 1788880241] req@ffffa06188698380 x1875771511724416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880241.790211:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511724416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880241.790230:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880241.790236:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880241.790243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880241.790257:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880241.790302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880241.790326:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880241/real 1788880241] req@ffffa0618869b800 x1875771511724544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880241.790352:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511724544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880241.790434:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880241.790441:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880241.790447:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880241.790539:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880241.790607:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880241.790638:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880241/real 1788880241] req@ffffa06188699c00 x1875771511724672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880241.790677:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511724672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880241.790695:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880241.790703:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880241.790714:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880241.790732:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880241.790777:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880241.790803:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880241/real 1788880241] req@ffffa0618869aa00 x1875771511724800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880241.790827:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511724800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880241.790845:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880241.790861:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880241.790868:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511724800:192.168.203.111@tcp:8:kworker.0 00010000:00010000:1.0:1788880242.429613:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880242.429622:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788880242.685530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880243.453520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880243.453643:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880243.709492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880244.477576:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880244.477666:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880244.733530:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880245.501577:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880245.501682:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880245.757477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880246.525478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880246.525528:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880246.781466:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880246.909579:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880246.909589:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880246.909593:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880246.909604:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5233 00000100:00080000:0.0:1788880246.909617:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880246.909628:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880246.909669:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511724928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880246.909691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880246.909694:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880246.909695:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880246.909698:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5233 00000100:00080000:0.0:1788880246.909701:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880246.909703:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880246.909711:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880246.909720:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511725056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880246.909729:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880246.909731:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880246.909733:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880246.909737:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5233 00000100:00080000:0.0:1788880246.909739:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788880246.909739:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511724928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880246.909746:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880246.909751:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880246.909759:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511725184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880246.909765:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880246.909768:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880246.909770:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880246.909773:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5233 00000100:00080000:0.0:1788880246.909775:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880246.909778:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880246.909782:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880246.909789:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511725312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880246.909795:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880246.909807:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880246.909809:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880246.909829:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5233 00000100:00080000:0.0:1788880246.909833:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880246.909836:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880246.909840:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880246.909847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880246.909852:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511725440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880246.909905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880246/real 1788880246] req@ffffa06198afea00 x1875771511724928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880246.909926:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511724928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880246.909937:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880246.909941:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880246.909945:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511724928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880246.909972:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880246.909996:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880246.910007:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880246/real 1788880246] req@ffffa06198afc000 x1875771511725056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880246.910017:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511725056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880246.910024:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880246.910027:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880246.910029:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880246.910034:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880246.910047:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880246.910054:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880246/real 1788880246] req@ffffa06198afc700 x1875771511725184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880246.910064:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511725184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880246.910071:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880246.910073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880246.910091:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880246.910096:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880246.910117:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880246.910130:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880246/real 1788880246] req@ffffa06198afdc00 x1875771511725312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880246.910143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511725312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880246.910152:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880246.910155:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880246.910175:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880246.910183:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880246.910209:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880246.910224:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880246/real 1788880246] req@ffffa06198afdf80 x1875771511725440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880246.910234:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511725440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880246.910247:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880246.910250:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880246.910253:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880247.549534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880247.549659:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880247.805562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880248.573492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880248.573677:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880248.829582:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880249.597541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880249.597669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880249.853536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880250.621592:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880250.621692:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880250.877565:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880251.645559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880251.645609:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880251.901558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880252.029527:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880252.029539:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880252.029544:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880252.029558:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5238 00000100:00080000:0.0:1788880252.029573:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880252.029584:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880252.029631:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511725568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880252.029669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880252.029674:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880252.029677:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880252.029683:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5238 00000100:00080000:0.0:1788880252.029687:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880252.029691:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880252.029704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880252.029722:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511725696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880252.029737:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880252.029742:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880252.029744:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880252.029749:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5238 00000100:00080000:0.0:1788880252.029752:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880252.029760:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880252.029775:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511725568:192.168.203.111@tcp:250:kworker.0 00000008:00100000:0.0:1788880252.029777:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880252.029810:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511725824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880252.029827:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880252.029834:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880252.029839:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880252.029845:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5238 00000100:00080000:0.0:1788880252.029849:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880252.029856:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880252.029866:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880252.029882:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511725952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880252.029895:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880252.029900:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880252.029903:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880252.029908:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5238 00000100:00080000:0.0:1788880252.029918:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788880252.029918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788880252.029939:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880252.029947:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880252.029966:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511726080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880252.030013:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880252/real 1788880252] req@ffffa06198aff800 x1875771511725568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880252.030056:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511725568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880252.030080:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880252.030093:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880252.030106:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511725568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880252.030129:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880252.030174:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880252.030193:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880252/real 1788880252] req@ffffa06198affb80 x1875771511725696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880252.030210:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511725696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880252.030222:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880252.030226:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880252.030231:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880252.030268:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880252.030289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880252.030301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880252/real 1788880252] req@ffffa06198afd880 x1875771511725824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880252.030316:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511725824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880252.030332:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880252.030338:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880252.030345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880252.030359:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880252.030486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880252.030507:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880252/real 1788880252] req@ffffa06198aff480 x1875771511725952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880252.030523:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511725952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880252.030534:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880252.030539:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880252.030543:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511725952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880252.030551:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880252.030587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880252.030611:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880252/real 1788880252] req@ffffa06198afed80 x1875771511726080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880252.030636:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511726080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880252.030654:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880252.030661:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880252.030669:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880252.669573:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880252.669600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880252.925524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880253.693534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880253.693616:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880253.949509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880254.717538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880254.717681:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880254.973509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880255.741519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880255.741639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880255.997465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880256.765471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880256.765535:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880257.021464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880257.149518:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880257.149526:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880257.149530:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880257.149539:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5243 00000100:00080000:2.0:1788880257.149548:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880257.149554:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880257.149583:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511726208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880257.149606:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880257.149609:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880257.149610:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880257.149613:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5243 00000100:00080000:2.0:1788880257.149616:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880257.149619:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880257.149628:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880257.149640:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511726336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880257.149641:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511726208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880257.149655:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880257.149658:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880257.149659:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880257.149662:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5243 00000100:00080000:2.0:1788880257.149665:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880257.149667:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880257.149670:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880257.149679:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511726464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880257.149690:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880257.149693:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880257.149694:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880257.149696:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5243 00000100:00080000:2.0:1788880257.149699:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880257.149701:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880257.149704:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880257.149708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880257.149713:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511726592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880257.149720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880257.149723:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880257.149725:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880257.149727:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5243 00000100:00080000:2.0:1788880257.149729:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880257.149732:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880257.149735:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880257.149743:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880257/real 1788880257] req@ffffa061888ad180 x1875771511726208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880257.149744:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511726720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880257.149779:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511726208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880257.149789:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880257.149792:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880257.149795:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511726208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880257.149812:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880257.149827:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880257.149835:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880257/real 1788880257] req@ffffa061888ace00 x1875771511726336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880257.149843:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511726336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880257.149849:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880257.149851:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880257.149853:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880257.149856:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880257.149866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880257.149872:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880257/real 1788880257] req@ffffa061888ae680 x1875771511726464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880257.149879:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511726464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880257.149884:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880257.149886:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880257.149888:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880257.149892:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880257.149902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880257.149908:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880257/real 1788880257] req@ffffa061888ad880 x1875771511726592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880257.149915:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511726592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880257.149920:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880257.149922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880257.149924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880257.149927:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880257.149936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880257.149941:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880257/real 1788880257] req@ffffa061888ac380 x1875771511726720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880257.149960:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511726720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880257.149964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880257.149966:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880257.149968:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880257.789455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880257.789521:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880258.045457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880258.813451:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880258.813518:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880259.069450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880259.837472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880259.837527:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880260.093458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880260.861509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880260.861683:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880261.117503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880261.885555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880261.885648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880262.141504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880262.269536:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880262.269546:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880262.269553:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880262.269567:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5248 00000100:00080000:2.0:1788880262.269583:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880262.269601:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880262.269649:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511726848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880262.269685:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880262.269690:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880262.269693:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880262.269699:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5248 00000100:00080000:2.0:1788880262.269705:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880262.269710:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880262.269722:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880262.269748:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511726976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880262.269765:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511726848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880262.269768:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880262.269782:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880262.269784:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880262.269791:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5248 00000100:00080000:2.0:1788880262.269796:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880262.269801:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880262.269809:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880262.269827:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511727104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880262.269839:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880262.269844:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880262.269847:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880262.269851:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5248 00000100:00080000:2.0:1788880262.269856:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880262.269861:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880262.269869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788880262.269874:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880262.269895:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511727232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880262.269909:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880262.269913:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880262.269917:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880262.269922:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5248 00000100:00100000:0.0:1788880262.269930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880262/real 1788880262] req@ffffa061888ac700 x1875771511726848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880262.269931:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880262.269944:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880262.269955:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880262.269973:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511727360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880262.269980:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511726848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880262.270022:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880262.270029:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880262.270036:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511726848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880262.270066:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880262.270094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880262.270109:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880262/real 1788880262] req@ffffa061888aed80 x1875771511726976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880262.270125:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511726976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880262.270137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880262.270141:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880262.270145:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511726976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880262.270152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880262.270179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880262.270199:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880262/real 1788880262] req@ffffa061888adc00 x1875771511727104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880262.270224:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511727104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880262.270256:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880262.270263:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880262.270270:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880262.270280:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880262.270311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880262.270357:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880262/real 1788880262] req@ffffa061888aea00 x1875771511727232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880262.270373:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511727232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880262.270413:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880262.270418:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880262.270422:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880262.270430:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880262.270457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880262.270472:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880262/real 1788880262] req@ffffa061888ae300 x1875771511727360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880262.270486:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511727360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880262.270497:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880262.270500:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880262.270505:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880262.909518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880262.909633:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880263.165501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880263.933578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880263.933621:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880264.189522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880264.957497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880264.957666:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880265.213481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880265.981474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880265.981651:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880266.237495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880267.005457:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880267.005513:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880267.261475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880267.389478:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880267.389486:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880267.389489:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880267.389498:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5254 00000100:00080000:0.0:1788880267.389509:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880267.389515:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880267.389550:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511727488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880267.389573:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880267.389576:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880267.389577:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880267.389580:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5254 00000100:00080000:0.0:1788880267.389583:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880267.389586:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880267.389594:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880267.389605:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511727616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880267.389608:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511727488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880267.389616:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880267.389620:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880267.389621:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880267.389623:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5254 00000100:00080000:0.0:1788880267.389626:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880267.389629:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880267.389632:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880267.389640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511727744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880267.389646:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880267.389649:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880267.389650:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880267.389653:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5254 00000100:00080000:0.0:1788880267.389655:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880267.389657:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880267.389660:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880267.389667:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511727872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880267.389672:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880267.389675:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880267.389676:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880267.389678:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5254 00000100:00080000:0.0:1788880267.389681:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880267.389683:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880267.389685:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880267.389688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880267.389695:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511728000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880267.389725:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880267/real 1788880267] req@ffffa06198afed80 x1875771511727488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880267.389745:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511727488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880267.389755:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880267.389759:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880267.389764:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511727488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880267.389784:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880267.389800:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880267.389808:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880267/real 1788880267] req@ffffa06198aff480 x1875771511727616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880267.389816:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511727616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880267.389822:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880267.389824:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880267.389826:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880267.389830:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880267.389839:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880267.389845:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880267/real 1788880267] req@ffffa06198afd880 x1875771511727744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880267.389852:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511727744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880267.389857:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880267.389859:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880267.389861:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880267.389865:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880267.389886:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880267.389892:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880267/real 1788880267] req@ffffa06198affb80 x1875771511727872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880267.389898:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511727872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880267.389903:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880267.389905:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880267.389907:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511727872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880267.389911:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880267.389921:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880267.389926:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880267/real 1788880267] req@ffffa06198aff800 x1875771511728000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880267.389933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511728000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880267.389938:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880267.389940:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880267.389942:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880268.029478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880268.029562:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880268.285476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880269.053489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880269.053546:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880269.309593:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880270.077534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880270.077583:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880270.333562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880271.101543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880271.101641:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880271.357558:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880272.125611:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880272.125686:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880272.381553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880272.509577:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880272.509593:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880272.509600:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880272.509619:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5259 00000100:00080000:2.0:1788880272.509637:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880272.509648:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880272.509698:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511728128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880272.509744:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880272.509749:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880272.509752:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880272.509758:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5259 00000100:00080000:2.0:1788880272.509762:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880272.509768:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880272.509784:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880272.509805:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511728256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880272.509827:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511728128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880272.509832:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880272.509854:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880272.509858:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880272.509865:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5259 00000100:00080000:2.0:1788880272.509870:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880272.509876:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880272.509884:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880272.509902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511728384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880272.509914:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880272.509920:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880272.509923:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880272.509928:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5259 00000100:00080000:2.0:1788880272.509934:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880272.509939:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880272.509946:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880272.509959:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880272.509962:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511728512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880272.509982:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880272.509988:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880272.509991:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880272.509999:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5259 00000100:00080000:2.0:1788880272.510005:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880272.510012:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880272.510022:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880272.510032:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880272/real 1788880272] req@ffffa061888af100 x1875771511728128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880272.510040:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511728640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880272.510140:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511728128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880272.510161:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880272.510171:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880272.510181:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511728128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880272.510232:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880272.510305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880272.510327:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880272/real 1788880272] req@ffffa061888afb80 x1875771511728256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880272.510346:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511728256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880272.510358:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880272.510363:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880272.510367:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880272.510376:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880272.510444:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880272.510459:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880272/real 1788880272] req@ffffa061888af480 x1875771511728384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880272.510475:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511728384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880272.510485:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880272.510490:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880272.510494:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880272.510502:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880272.510524:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880272.510535:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880272/real 1788880272] req@ffffa061888adf80 x1875771511728512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880272.510549:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511728512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880272.510560:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880272.510564:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880272.510567:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880272.510575:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880272.510594:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880272.510606:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880272/real 1788880272] req@ffffa061888aca80 x1875771511728640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880272.510630:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511728640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880272.510640:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880272.510644:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880272.510648:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880273.149546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880273.149623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880273.405548:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880274.173565:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880274.173653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880274.429549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880275.197560:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880275.197713:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880275.453590:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880276.221520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880276.221639:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880276.477515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880277.245500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880277.245679:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880277.501459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880277.629531:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880277.629545:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880277.629552:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880277.629571:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5264 00000100:00080000:2.0:1788880277.629591:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880277.629601:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880277.629660:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511728768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880277.629726:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880277.629734:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880277.629738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880277.629750:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5264 00000100:00080000:2.0:1788880277.629760:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880277.629768:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880277.629785:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880277.629799:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511728768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880277.629815:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511728896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880277.629837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880277.629843:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880277.629848:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880277.629853:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5264 00000100:00080000:2.0:1788880277.629859:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880277.629863:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880277.629870:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880277.629887:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511729024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880277.629901:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880277.629905:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880277.629907:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880277.629913:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5264 00000100:00080000:2.0:1788880277.629917:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880277.629923:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880277.629931:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880277.629937:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880277.629954:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511729152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880277.629972:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880277.629978:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880277.629983:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880277.629993:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5264 00000100:00080000:2.0:1788880277.630000:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880277.630009:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880277.630018:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880277/real 1788880277] req@ffffa061888ad500 x1875771511728768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788880277.630019:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880277.630051:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511729280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880277.630069:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511728768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880277.630105:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880277.630113:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880277.630120:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511728768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880277.630179:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880277.630239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880277.630267:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880277/real 1788880277] req@ffffa061888ac000 x1875771511728896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880277.630293:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511728896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880277.630323:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880277.630330:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880277.630338:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511728896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880277.630351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880277.630422:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880277.630441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880277/real 1788880277] req@ffffa061888aca80 x1875771511729024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880277.630457:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511729024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880277.630468:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880277.630472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880277.630476:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880277.630484:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880277.630509:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880277.630521:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880277/real 1788880277] req@ffffa061888adf80 x1875771511729152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880277.630535:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511729152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880277.630545:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880277.630548:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880277.630554:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880277.630562:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880277.630582:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880277.630593:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880277/real 1788880277] req@ffffa061888af480 x1875771511729280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880277.630607:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511729280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880277.630618:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880277.630621:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880277.630625:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880278.269554:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880278.269656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880278.525547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880279.293546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880279.293629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880279.549500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880280.317477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880280.317581:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880280.573464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880281.341511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880281.341674:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880281.597516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880282.365529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880282.365688:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880282.621516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880282.749510:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880282.749520:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880282.749526:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880282.749543:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5269 00000100:00080000:2.0:1788880282.749566:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880282.749579:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880282.749634:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511729408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880282.749675:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880282.749679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880282.749682:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880282.749687:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5269 00000100:00080000:2.0:1788880282.749692:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880282.749697:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880282.749712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880282.749729:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511729536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880282.749752:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511729408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880282.749757:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880282.749779:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880282.749785:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880282.749792:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5269 00000100:00080000:2.0:1788880282.749795:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880282.749802:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880282.749810:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880282.749836:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511729664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880282.749855:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880282.749865:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880282.749869:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880282.749879:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5269 00000800:00000100:0.0:1788880282.749888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880282.749897:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880282.749913:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880282.749923:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880282.749951:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511729792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880282.749955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880282/real 1788880282] req@ffffa061888afb80 x1875771511729408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880282.749998:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880282.750003:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880282.750005:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880282.750028:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5269 00000100:00080000:2.0:1788880282.750041:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880282.750051:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880282.750058:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880282.750085:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511729920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880282.750088:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511729408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880282.750197:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880282.750211:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880282.750224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511729408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880282.750279:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880282.750338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880282.750359:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880282/real 1788880282] req@ffffa061888af100 x1875771511729536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880282.750424:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511729536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880282.750446:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880282.750454:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880282.750463:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880282.750478:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880282.750524:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880282.750549:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880282/real 1788880282] req@ffffa061888ae300 x1875771511729664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880282.750577:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511729664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880282.750592:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880282.750597:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880282.750602:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880282.750611:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880282.750643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880282.750659:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880282/real 1788880282] req@ffffa061888aea00 x1875771511729792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880282.750675:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511729792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880282.750686:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880282.750690:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880282.750694:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880282.750701:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880282.750721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880282.750732:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880282/real 1788880282] req@ffffa061888adc00 x1875771511729920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880282.750746:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511729920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880282.750757:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880282.750761:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880282.750765:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511729920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880283.389545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880283.389643:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880283.645551:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880284.413585:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880284.413675:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880284.669526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880285.437521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880285.437689:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880285.693457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880286.461481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880286.461579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880286.717484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880287.485504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880287.485593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880287.741525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880287.869553:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880287.869562:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880287.869567:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880287.869581:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5274 00000100:00080000:1.0:1788880287.869595:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880287.869605:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880287.869645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511730048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880287.869665:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880287.869668:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880287.869670:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880287.869674:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5274 00000100:00080000:1.0:1788880287.869677:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880287.869680:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880287.869691:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880287.869706:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511730176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880287.869708:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511730048:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880287.869739:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880287.869742:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880287.869743:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880287.869747:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5274 00000100:00080000:1.0:1788880287.869751:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880287.869753:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880287.869757:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880287.869768:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511730304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880287.869775:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880287.869778:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880287.869780:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880287.869783:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5274 00000100:00080000:1.0:1788880287.869786:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880287.869789:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880287.869795:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880287.869803:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511730432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880287.869810:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880287.869813:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880287.869814:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880287.869816:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5274 00000100:00080000:1.0:1788880287.869819:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880287.869822:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880287.869826:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880287.869835:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880287.869843:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511730560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880287.869897:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880287/real 1788880287] req@ffffa0618869aa00 x1875771511730048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880287.869935:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511730048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880287.869950:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880287.869956:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880287.869962:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511730048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880287.869995:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880287.870029:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880287.870053:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880287/real 1788880287] req@ffffa06188699c00 x1875771511730176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880287.870078:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511730176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880287.870097:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880287.870104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880287.870111:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880287.870124:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880287.870192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880287.870210:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880287/real 1788880287] req@ffffa0618869b800 x1875771511730304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880287.870225:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511730304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880287.870244:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880287.870248:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880287.870252:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880287.870260:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880287.870284:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880287.870296:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880287/real 1788880287] req@ffffa06188698380 x1875771511730432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880287.870310:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511730432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880287.870320:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880287.870323:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880287.870327:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880287.870334:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880287.870354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880287.870366:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880287/real 1788880287] req@ffffa0618869b100 x1875771511730560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880287.870413:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511730560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880287.870424:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880287.870428:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880287.870434:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880288.509474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880288.509736:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880288.765511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880289.533481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880289.533529:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880289.789485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880290.557472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880290.557678:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880290.813481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880291.581461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880291.581658:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880291.837488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880292.605567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880292.605733:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880292.861547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880292.989638:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880292.989649:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880292.989655:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880292.989668:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5279 00000100:00080000:2.0:1788880292.989682:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880292.989691:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880292.989741:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511730688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880292.989781:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880292.989786:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880292.989789:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880292.989794:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5279 00000100:00080000:2.0:1788880292.989798:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880292.989803:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880292.989816:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880292.989833:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511730816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880292.989849:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880292.989856:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880292.989860:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880292.989868:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5279 00000100:00100000:0.0:1788880292.989873:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511730688:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880292.989873:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880292.989881:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880292.989903:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880292.989929:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511730944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880292.989943:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880292.989947:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880292.989950:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880292.989958:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5279 00000100:00080000:2.0:1788880292.989966:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880292.989972:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880292.989979:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880292.989993:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511731072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880292.990014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880292.990015:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880292.990031:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880292.990034:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880292.990041:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5279 00000100:00080000:2.0:1788880292.990050:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880292.990058:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880292.990065:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880292.990089:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511731200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880292.990094:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880292/real 1788880292] req@ffffa061888aed80 x1875771511730688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880292.990205:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511730688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880292.990225:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880292.990231:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880292.990237:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511730688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880292.990278:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880292.990317:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880292.990343:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880292/real 1788880292] req@ffffa061888ac700 x1875771511730816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880292.990369:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511730816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880292.990435:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880292.990441:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880292.990446:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880292.990461:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880292.990495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880292.990512:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880292/real 1788880292] req@ffffa061888ac380 x1875771511730944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880292.990528:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511730944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880292.990538:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880292.990542:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880292.990546:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511730944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880292.990554:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880292.990575:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880292.990588:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880292/real 1788880292] req@ffffa061888ad880 x1875771511731072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880292.990601:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511731072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880292.990612:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880292.990616:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880292.990620:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880292.990627:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880292.990646:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880292.990658:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880292/real 1788880292] req@ffffa061888ae680 x1875771511731200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880292.990673:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511731200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880292.990683:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880292.990687:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880292.990690:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880293.629501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880293.629680:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880293.885476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880294.653511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880294.653596:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880294.909520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880295.677520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880295.677655:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880295.933502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880296.701484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880296.701655:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880296.957477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880297.725481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880297.725651:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880297.981507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880298.109589:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880298.109599:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880298.109604:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880298.109618:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5284 00000100:00080000:1.0:1788880298.109633:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880298.109642:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880298.109689:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511731328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880298.109729:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880298.109734:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880298.109737:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880298.109743:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5284 00000100:00080000:1.0:1788880298.109747:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880298.109752:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880298.109766:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880298.109790:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511731456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880298.109791:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511731328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880298.109820:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880298.109824:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880298.109827:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880298.109833:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5284 00000100:00080000:1.0:1788880298.109837:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880298.109841:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880298.109847:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880298.109863:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511731584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880298.109874:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880298.109878:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880298.109882:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880298.109885:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5284 00000100:00080000:1.0:1788880298.109890:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880298.109893:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880298.109901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788880298.109902:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880298.109923:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511731712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880298.109936:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880298.109941:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880298.109944:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880298.109950:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5284 00000100:00100000:0.0:1788880298.109958:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880298/real 1788880298] req@ffffa06188699f80 x1875771511731328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880298.109960:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880298.109972:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880298.109983:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880298.110002:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511731840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880298.110010:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511731328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880298.110046:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880298.110052:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880298.110058:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511731328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880298.110124:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880298.110157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880298.110175:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880298/real 1788880298] req@ffffa06188698000 x1875771511731456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880298.110201:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511731456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880298.110213:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880298.110217:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880298.110222:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880298.110228:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880298.110248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880298.110259:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880298/real 1788880298] req@ffffa0618869a680 x1875771511731584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880298.110273:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511731584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880298.110283:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880298.110286:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880298.110290:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880298.110297:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880298.110316:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880298.110327:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880298/real 1788880298] req@ffffa0618869bb80 x1875771511731712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880298.110342:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511731712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880298.110371:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880298.110377:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880298.110423:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880298.110437:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880298.110476:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880298.110493:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880298/real 1788880298] req@ffffa0618869ad80 x1875771511731840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880298.110508:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511731840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880298.110519:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880298.110523:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880298.110527:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511731840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880298.749495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880298.749652:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880299.005512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880299.773555:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880299.773642:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880300.029476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880300.797520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880300.797724:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880301.053516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880301.821534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880301.823429:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880302.077534:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880302.845538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880302.845639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880303.101500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880303.229589:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880303.229600:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880303.229605:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880303.229618:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5289 00000100:00080000:2.0:1788880303.229634:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880303.229644:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880303.229695:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511731968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880303.229735:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880303.229741:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880303.229744:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880303.229750:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5289 00000100:00080000:2.0:1788880303.229754:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880303.229760:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880303.229773:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880303.229790:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511732096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880303.229808:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880303.229813:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511731968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880303.229815:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880303.229822:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880303.229844:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5289 00000100:00080000:2.0:1788880303.229849:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880303.229853:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880303.229861:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880303.229879:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511732224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880303.229890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880303.229894:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880303.229896:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880303.229901:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5289 00000100:00080000:2.0:1788880303.229905:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880303.229910:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880303.229916:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880303.229931:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511732352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880303.229942:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880303.229948:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880303.229952:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880303.229957:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5289 00000100:00080000:2.0:1788880303.229962:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880303.229968:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880303.229975:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880303.229988:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511732480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880303.230067:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880303.230121:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880303.230174:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880303.230225:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880303.869569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880303.869737:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880304.125516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880304.893545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880304.893717:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880305.149497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880305.917497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880305.917679:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880306.173532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880306.301788:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880306.301840:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880306.301863:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880306.301911:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880306.301922:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880306.301931:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880306.301941:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788880306.302074:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880303/real 1788880306] req@ffffa061888ad180 x1875771511732096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880306.302123:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511732096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880306.302148:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880306.302157:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880306.302166:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880306.302182:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880303/real 1788880306] req@ffffa061888ae680 x1875771511732224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880306.302208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511732224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880306.302226:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880306.302233:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880306.302239:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880306.302249:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880303/real 1788880306] req@ffffa061888ad880 x1875771511732352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880306.302265:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511732352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880306.302275:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880306.302281:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880306.302288:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880306.302301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880303/real 1788880306] req@ffffa061888ac380 x1875771511732480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880306.302329:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511732480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880306.302347:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880306.302355:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880306.302363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880306.302376:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880303/real 1788880306] req@ffffa061888ace00 x1875771511731968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880306.302449:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511731968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880306.302467:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880306.302476:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880306.302484:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511731968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880306.941566:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880306.941623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880307.197585:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880307.965531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880307.965615:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880308.221497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880308.349584:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880308.349598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880308.349605:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880308.349625:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5295 00000100:00080000:2.0:1788880308.349644:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880308.349657:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880308.349711:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511732608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880308.349748:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880308.349753:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880308.349755:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880308.349760:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5295 00000100:00080000:2.0:1788880308.349766:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880308.349774:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880308.349796:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880308.349814:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511732736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880308.349839:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880308.349842:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511732608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880308.349845:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880308.349850:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880308.349868:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5295 00000100:00080000:2.0:1788880308.349881:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880308.349890:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880308.349901:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880308.349928:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511732864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880308.349962:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880308.349970:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880308.349974:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880308.349983:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5295 00000100:00080000:2.0:1788880308.350023:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880308.350035:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880308.350048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788880308.350052:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880308.350078:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511732992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880308.350101:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880308.350110:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880308.350114:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880308.350132:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5295 00000100:00100000:0.0:1788880308.350135:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880308/real 1788880308] req@ffffa061888ac700 x1875771511732608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880308.350143:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880308.350164:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880308.350182:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880308.350209:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511733120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880308.350237:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511732608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880308.350271:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880308.350282:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880308.350290:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511732608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880308.350340:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732736:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880308.350444:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880308.350473:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880308/real 1788880308] req@ffffa061888aed80 x1875771511732736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880308.350505:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511732736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880308.350524:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880308.350532:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880308.350539:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880308.350553:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880308.350599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880308.350624:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880308/real 1788880308] req@ffffa061888adc00 x1875771511732864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880308.350641:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511732864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880308.350661:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880308.350665:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880308.350669:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880308.350678:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880308.350706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880308.350718:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880308/real 1788880308] req@ffffa061888aea00 x1875771511732992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880308.350733:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511732992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880308.350744:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880308.350748:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880308.350751:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511732992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880308.350759:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880308.350777:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880308.350789:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880308/real 1788880308] req@ffffa061888ae300 x1875771511733120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880308.350803:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511733120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880308.350813:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880308.350817:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880308.350822:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880308.989529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880308.989659:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880309.245489:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880310.013500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880310.013649:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880310.269507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880311.037509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880311.037671:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880311.293520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880312.061537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880312.061646:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880312.317510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880313.085531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880313.085603:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880313.341516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880313.469582:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880313.469595:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880313.469602:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880313.469618:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5300 00000100:00080000:3.0:1788880313.469634:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880313.469644:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880313.469701:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511733248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880313.469742:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880313.469747:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880313.469750:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880313.469755:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5300 00000100:00080000:3.0:1788880313.469759:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880313.469765:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880313.469781:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880313.469805:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511733248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880313.469806:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511733376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880313.469842:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880313.469848:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880313.469850:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880313.469855:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5300 00000100:00080000:3.0:1788880313.469859:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880313.469863:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880313.469870:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880313.469887:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511733504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880313.469899:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880313.469904:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880313.469906:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880313.469911:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5300 00000100:00080000:3.0:1788880313.469915:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880313.469923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880313.469927:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880313.469936:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880313.469952:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511733632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880313.469996:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880313.470001:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880313.470010:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880313.470011:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880313/real 1788880313] req@ffffa06188870a80 x1875771511733248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880313.470025:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5300 00000100:00080000:3.0:1788880313.470037:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880313.470047:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880313.470055:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880313.470061:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511733248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880313.470082:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788880313.470083:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511733760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880313.470093:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880313.470108:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511733248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880313.470151:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880313.470186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880313.470203:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880313/real 1788880313] req@ffffa06188872d80 x1875771511733376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880313.470220:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511733376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880313.470231:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880313.470235:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880313.470241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880313.470249:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880313.470269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880313.470280:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880313/real 1788880313] req@ffffa06188872300 x1875771511733504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880313.470303:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511733504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880313.470320:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880313.470326:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880313.470333:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880313.470345:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880313.470423:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880313.470443:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880313/real 1788880313] req@ffffa06188873b80 x1875771511733632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880313.470459:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511733632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880313.470470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880313.470474:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880313.470478:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880313.470487:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880313.470513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880313.470526:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880313/real 1788880313] req@ffffa06188872a00 x1875771511733760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880313.470541:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511733760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880313.470551:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880313.470554:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880313.470558:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511733760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880314.109533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880314.109630:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880314.365562:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880315.133490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880315.133672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880315.389505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880316.157510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880316.157629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880316.413485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880317.181441:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880317.181588:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880317.437440:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880318.205567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880318.205649:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880318.461505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880318.589543:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880318.589552:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880318.589558:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880318.589574:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5305 00000100:00080000:1.0:1788880318.589588:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880318.589599:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880318.589675:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511733888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880318.589723:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880318.589729:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880318.589732:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880318.589738:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5305 00000100:00080000:1.0:1788880318.589745:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880318.589752:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880318.589768:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880318.589811:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511733888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880318.589816:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511734016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880318.589830:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880318.589834:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880318.589836:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880318.589843:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5305 00000100:00080000:1.0:1788880318.589847:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880318.589852:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880318.589860:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880318.589878:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511734144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880318.589890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880318.589894:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880318.589900:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880318.589902:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880318.589907:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5305 00000100:00080000:1.0:1788880318.589914:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880318.589918:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880318.589940:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880318.589946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880318/real 1788880318] req@ffffa0618869ad80 x1875771511733888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880318.589956:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511734272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880318.589966:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880318.589971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511733888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880318.589972:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880318.589977:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880318.589985:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5305 00000100:00080000:1.0:1788880318.589993:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880318.589995:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880318.589997:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880318.590005:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880318.590006:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880318.590013:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511733888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880318.590019:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511734400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880318.590045:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880318.590065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880318.590076:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880318/real 1788880318] req@ffffa0618869bb80 x1875771511734016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880318.590085:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511734016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880318.590092:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880318.590095:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880318.590097:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880318.590101:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880318.590114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880318.590121:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880318/real 1788880318] req@ffffa0618869a680 x1875771511734144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880318.590129:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511734144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880318.590134:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880318.590136:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880318.590138:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880318.590142:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880318.590154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880318.590161:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880318/real 1788880318] req@ffffa06188698000 x1875771511734272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880318.590169:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511734272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880318.590205:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880318.590207:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880318.590210:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880318.590214:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880318.590228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880318.590237:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880318/real 1788880318] req@ffffa06188699f80 x1875771511734400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880318.590252:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511734400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880318.590262:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880318.590264:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880318.590266:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880319.229553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880319.229644:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880319.485506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880320.253533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880320.253583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880320.509525:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880321.277533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880321.277602:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880321.533497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880322.301526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880322.301656:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880322.557503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880323.325521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880323.325616:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880323.581484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880323.709608:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880323.709618:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880323.709625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880323.709639:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5310 00000100:00080000:3.0:1788880323.709654:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880323.709664:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880323.709721:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511734528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880323.709762:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880323.709767:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880323.709770:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880323.709775:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5310 00000100:00080000:3.0:1788880323.709780:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880323.709785:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880323.709800:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880323.709818:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511734656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880323.709830:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511734528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880323.709840:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880323.709851:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880323.709853:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880323.709859:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5310 00000100:00080000:3.0:1788880323.709864:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880323.709869:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880323.709875:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880323.709918:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511734784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880323.709932:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880323.709936:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880323.709938:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880323.709944:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5310 00000100:00080000:3.0:1788880323.709949:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880323.709953:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880323.709960:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880323.709968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880323.709983:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511734912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880323.709999:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880323.710003:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880323.710006:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880323.710011:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5310 00000100:00080000:3.0:1788880323.710017:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880323.710022:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880323.710029:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880323.710037:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880323/real 1788880323] req@ffffa06188870380 x1875771511734528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880323.710052:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511735040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880323.710079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511734528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880323.710101:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880323.710111:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880323.710120:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511734528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880323.710164:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880323.710211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880323.710233:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880323/real 1788880323] req@ffffa06188871880 x1875771511734656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880323.710250:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511734656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880323.710262:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880323.710266:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880323.710270:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880323.710279:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880323.710308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880323.710328:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880323/real 1788880323] req@ffffa06188873800 x1875771511734784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880323.710353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511734784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880323.710371:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880323.710378:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880323.710422:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880323.710432:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880323.710466:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880323.710482:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880323/real 1788880323] req@ffffa06188871c00 x1875771511734912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880323.710498:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511734912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880323.710509:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880323.710513:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880323.710517:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511734912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880323.710526:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880323.710562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880323.710584:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880323/real 1788880323] req@ffffa06188872a00 x1875771511735040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880323.710608:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511735040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880323.710626:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880323.710633:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880323.710637:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880324.349479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880324.349612:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880324.605477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880325.373509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880325.373634:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880325.629474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880326.397490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880326.397612:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880326.653473:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880327.421464:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880327.421663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880327.677533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880328.445502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880328.445665:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880328.701527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880328.829524:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880328.829536:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880328.829543:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880328.829560:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5315 00000100:00080000:3.0:1788880328.829577:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880328.829588:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880328.829647:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511735168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880328.829685:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880328.829691:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880328.829694:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880328.829699:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5315 00000100:00080000:3.0:1788880328.829704:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880328.829709:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880328.829725:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880328.829751:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511735296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880328.829752:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511735168:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880328.829780:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880328.829785:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880328.829787:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880328.829792:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5315 00000100:00080000:3.0:1788880328.829796:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880328.829801:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880328.829808:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880328.829823:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511735424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880328.829836:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880328.829840:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880328.829843:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880328.829870:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5315 00000100:00080000:3.0:1788880328.829877:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880328.829883:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880328.829890:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880328.829907:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511735552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880328.829920:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788880328.829928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880328.829930:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880328.829938:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880328.829945:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5315 00000100:00080000:3.0:1788880328.829951:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880328.829956:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880328.829963:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880328.829982:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511735680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880328.829999:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880328/real 1788880328] req@ffffa06188873b80 x1875771511735168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880328.830034:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511735168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880328.830052:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880328.830058:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880328.830064:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511735168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880328.830098:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880328.830130:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880328.830147:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880328/real 1788880328] req@ffffa06188872300 x1875771511735296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880328.830163:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511735296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880328.830174:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880328.830178:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880328.830182:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880328.830189:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880328.830210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880328.830222:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880328/real 1788880328] req@ffffa06188872d80 x1875771511735424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880328.830237:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511735424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880328.830247:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880328.830250:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880328.830254:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880328.830261:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880328.830281:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880328.830293:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880328/real 1788880328] req@ffffa06188870a80 x1875771511735552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880328.830307:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511735552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880328.830317:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880328.830320:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880328.830323:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880328.830330:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880328.830349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880328.830360:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880328/real 1788880328] req@ffffa06188873480 x1875771511735680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880328.830373:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511735680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880328.830426:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880328.830430:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880328.830434:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880329.469486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880329.469565:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880329.725445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880330.493504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880330.493669:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880330.749486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880331.517526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880331.517650:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880331.773509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880332.541492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880332.541640:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880332.797493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880333.565521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880333.565672:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880333.821501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880333.949582:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880333.949594:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880333.949601:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880333.949617:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5320 00000100:00080000:1.0:1788880333.949634:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880333.949646:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880333.949704:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511735808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880333.949745:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880333.949750:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880333.949752:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880333.949758:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5320 00000100:00080000:1.0:1788880333.949762:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880333.949767:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880333.949781:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880333.949800:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511735936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880333.949836:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880333.949843:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511735808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880333.949844:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880333.949851:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880333.949865:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5320 00000100:00080000:1.0:1788880333.949869:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880333.949874:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880333.949881:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880333.949902:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511736064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880333.949915:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880333.949920:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880333.949923:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880333.949927:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5320 00000100:00080000:1.0:1788880333.949943:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880333.949948:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880333.949955:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880333.949962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880333.949980:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511736192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880333.949994:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880333.949999:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880333.950002:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880333.950009:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5320 00000100:00080000:1.0:1788880333.950016:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880333.950025:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880333/real 1788880333] req@ffffa0618869b100 x1875771511735808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880333.950026:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880333.950040:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880333.950062:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511736320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880333.950077:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511735808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880333.950108:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880333.950118:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880333.950129:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511735808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880333.950176:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880333.950214:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880333.950233:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880333/real 1788880333] req@ffffa06188698380 x1875771511735936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880333.950250:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511735936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880333.950262:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880333.950267:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880333.950271:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511735936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880333.950280:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880333.950301:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880333.950312:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880333/real 1788880333] req@ffffa0618869b800 x1875771511736064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880333.950327:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511736064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880333.950337:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880333.950341:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880333.950345:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880333.950352:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880333.950372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880333.950416:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880333/real 1788880333] req@ffffa06188699c00 x1875771511736192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880333.950432:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511736192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880333.950443:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880333.950447:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880333.950452:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880333.950460:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880333.950484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880333.950498:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880333/real 1788880333] req@ffffa0618869aa00 x1875771511736320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880333.950512:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511736320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880333.950522:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880333.950526:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880333.950529:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880334.589513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880334.589685:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880334.845543:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880335.613521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880335.613669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880335.869498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880336.637515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880336.637578:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880336.893511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880337.661541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880337.661685:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880337.917480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880338.685512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880338.685606:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880338.941500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880339.069516:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880339.069528:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880339.069535:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880339.069550:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5325 00000100:00080000:1.0:1788880339.069567:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880339.069579:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880339.069629:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511736448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880339.069666:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880339.069672:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880339.069676:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880339.069683:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5325 00000100:00080000:1.0:1788880339.069688:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880339.069694:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880339.069705:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511736448:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788880339.069708:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880339.069736:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511736576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880339.069751:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880339.069756:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880339.069760:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880339.069764:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5325 00000100:00080000:1.0:1788880339.069841:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880339.069848:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880339.069857:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788880339.069859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880339.069882:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511736704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880339.069896:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880339.069901:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880339.069904:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880339.069911:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5325 00000100:00080000:1.0:1788880339.069916:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880339.069926:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880339.069928:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880339/real 1788880339] req@ffffa0618869aa00 x1875771511736448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788880339.069933:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880339.069956:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511736832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880339.069969:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880339.069970:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511736448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880339.069980:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880339.069987:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880339.070001:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5325 00000100:00080000:1.0:1788880339.070007:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880339.070013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788880339.070014:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788880339.070033:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000008:00100000:1.0:1788880339.070040:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880339.070047:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511736448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880339.070061:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511736960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880339.070096:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880339.070147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880339.070172:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880339/real 1788880339] req@ffffa06188699c00 x1875771511736576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880339.070198:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511736576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880339.070211:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880339.070215:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880339.070219:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880339.070228:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880339.070255:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880339.070268:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880339/real 1788880339] req@ffffa0618869b800 x1875771511736704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880339.070282:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511736704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880339.070291:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880339.070295:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880339.070299:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880339.070305:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880339.070323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880339.070334:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880339/real 1788880339] req@ffffa06188698380 x1875771511736832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880339.070348:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511736832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880339.070357:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880339.070360:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880339.070363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880339.070370:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880339.070387:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880339.070397:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880339/real 1788880339] req@ffffa0618869b100 x1875771511736960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880339.070443:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511736960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880339.070453:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880339.070457:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880339.070461:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511736960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880339.709517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880339.709646:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880339.965481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880340.733502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880340.733689:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880340.989434:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880341.757506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880341.757679:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880342.013490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880342.781512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880342.781701:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880343.037503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880343.805471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880343.805687:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880344.061520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880344.189565:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880344.189577:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880344.189583:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880344.189597:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5330 00000100:00080000:3.0:1788880344.189613:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880344.189623:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880344.189671:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511737088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880344.189711:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880344.189716:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880344.189719:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880344.189725:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5330 00000100:00080000:3.0:1788880344.189730:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880344.189736:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880344.189778:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880344.189790:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511737088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880344.189808:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511737216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880344.189833:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880344.189841:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880344.189845:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880344.189854:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5330 00000100:00080000:3.0:1788880344.189863:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880344.189872:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880344.189885:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788880344.189887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880344.189922:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511737344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880344.189940:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880344.189945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880344/real 1788880344] req@ffffa06188871500 x1875771511737088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880344.189947:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880344.189953:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880344.189969:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5330 00000100:00080000:3.0:1788880344.189981:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880344.189990:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880344.189999:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880344.190001:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511737088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880344.190020:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511737472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880344.190021:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880344.190039:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880344.190054:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511737088:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880344.190055:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880344.190068:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880344.190073:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880344.190082:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5330 00000100:00080000:3.0:1788880344.190086:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880344.190091:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880344.190097:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880344.190110:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511737600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880344.190120:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880344.190180:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880344.190204:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880344/real 1788880344] req@ffffa06188873100 x1875771511737216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880344.190221:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511737216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880344.190233:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880344.190237:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880344.190242:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880344.190250:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880344.190276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880344.190289:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880344/real 1788880344] req@ffffa06188872680 x1875771511737344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880344.190303:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511737344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880344.190321:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880344.190326:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880344.190330:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880344.190336:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880344.190356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880344.190368:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880344/real 1788880344] req@ffffa06188870700 x1875771511737472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880344.190413:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511737472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880344.190425:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880344.190429:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880344.190433:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880344.190456:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880344.190479:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880344.190493:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880344/real 1788880344] req@ffffa06188871500 x1875771511737600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880344.190508:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511737600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880344.190517:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880344.190521:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880344.190526:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880344.829552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880344.829671:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880345.085484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:0.0:1788880345.853674:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880345.853805:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788880346.109483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880346.877557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880346.877710:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880347.133510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880347.901533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880347.901627:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880348.157447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880348.925486:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880348.925592:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880349.181459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880349.309524:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880349.309532:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880349.309536:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880349.309545:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5336 00000100:00080000:1.0:1788880349.309555:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880349.309562:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880349.309597:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511737728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880349.309621:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880349.309624:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880349.309625:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880349.309628:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5336 00000100:00080000:1.0:1788880349.309631:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880349.309634:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880349.309643:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880349.309654:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511737856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880349.309667:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511737728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880349.309667:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880349.309681:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880349.309684:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880349.309688:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5336 00000100:00080000:1.0:1788880349.309691:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880349.309693:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880349.309696:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880349.309718:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511737984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880349.309725:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880349.309728:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880349.309729:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880349.309732:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5336 00000100:00080000:1.0:1788880349.309735:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880349.309738:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880349.309744:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880349.309754:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511738112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880349.309760:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880349.309763:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880349.309764:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880349.309769:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5336 00000800:00000100:0.0:1788880349.309772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880349.309775:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880349.309780:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880349.309784:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880349.309795:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511738240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880349.309823:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880349/real 1788880349] req@ffffa06188699f80 x1875771511737728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880349.309840:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511737728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880349.309850:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880349.309854:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880349.309857:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511737728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880349.309881:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880349.309897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880349.309905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880349/real 1788880349] req@ffffa06188698000 x1875771511737856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880349.309913:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511737856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880349.309919:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880349.309922:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880349.309925:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880349.309930:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880349.309941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880349.309947:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880349/real 1788880349] req@ffffa0618869a680 x1875771511737984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880349.309954:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511737984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880349.309960:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880349.309961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880349.309963:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511737984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880349.309968:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880349.309983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880349.309995:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880349/real 1788880349] req@ffffa0618869bb80 x1875771511738112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880349.310009:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511738112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880349.310018:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880349.310022:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880349.310026:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880349.310033:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880349.310049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880349.310058:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880349/real 1788880349] req@ffffa0618869ad80 x1875771511738240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880349.310067:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511738240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880349.310072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880349.310075:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880349.310077:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880349.949463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880349.949555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880350.205533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880350.973562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880350.973669:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880351.229491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880351.997508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880351.997657:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880352.253471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880353.021515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880353.021587:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880353.277505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880354.045578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880354.045700:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880354.301501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880354.429534:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880354.429550:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880354.429559:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880354.429579:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5341 00000100:00080000:1.0:1788880354.429600:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880354.429614:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880354.429703:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511738368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880354.429750:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880354.429757:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880354.429760:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880354.429768:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5341 00000100:00080000:1.0:1788880354.429777:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880354.429787:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880354.429806:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880354.429814:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511738368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880354.429839:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511738496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880354.429858:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880354.429864:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880354.429867:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880354.429872:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5341 00000100:00080000:1.0:1788880354.429877:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880354.429882:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880354.429889:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880354.429905:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511738624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880354.429918:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880354.429923:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880354.429925:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880354.429935:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5341 00000800:00000100:0.0:1788880354.429936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880354.429941:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880354.429949:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880354.429956:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880354.429972:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511738752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880354.429986:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880354.429990:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880354.429993:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880354.429998:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880354/real 1788880354] req@ffffa0618869ad80 x1875771511738368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880354.430004:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5341 00000100:00080000:1.0:1788880354.430016:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880354.430028:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880354.430038:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880354.430048:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511738368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880354.430057:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511738880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880354.430098:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880354.430112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880354.430124:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511738368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880354.430169:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880354.430214:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880354.430234:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880354/real 1788880354] req@ffffa0618869bb80 x1875771511738496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880354.430267:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511738496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880354.430279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880354.430283:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880354.430288:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880354.430297:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880354.430321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880354.430334:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880354/real 1788880354] req@ffffa0618869a680 x1875771511738624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880354.430349:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511738624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880354.430359:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880354.430362:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880354.430366:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880354.430374:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880354.430430:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880354.430444:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880354/real 1788880354] req@ffffa06188698000 x1875771511738752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880354.430459:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511738752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880354.430470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880354.430474:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880354.430478:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880354.430485:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880354.430505:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880354.430517:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880354/real 1788880354] req@ffffa06188699f80 x1875771511738880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880354.430531:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511738880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880354.430541:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880354.430544:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880354.430548:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511738880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880355.069538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880355.069655:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880355.325458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880356.093451:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880356.093538:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880356.349522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880357.117511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880357.117577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880357.373498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880358.141445:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880358.141562:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880358.397444:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880359.165456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880359.165497:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880359.421483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880359.549480:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880359.549491:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880359.549495:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880359.549506:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5346 00000100:00080000:0.0:1788880359.549518:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880359.549526:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880359.549569:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511739008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880359.549601:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880359.549604:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880359.549605:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880359.549609:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5346 00000100:00080000:0.0:1788880359.549611:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880359.549613:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880359.549623:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880359.549652:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511739136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880359.549661:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880359.549664:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880359.549666:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880359.549669:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5346 00000100:00080000:0.0:1788880359.549672:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880359.549677:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880359.549677:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511739008:192.168.203.111@tcp:250:kworker.0 00000008:00100000:0.0:1788880359.549684:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880359.549696:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511739264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880359.549703:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880359.549706:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880359.549708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880359.549711:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5346 00000100:00080000:0.0:1788880359.549713:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880359.549715:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880359.549719:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880359.549730:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511739392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880359.549736:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880359.549738:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880359.549739:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880359.549741:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5346 00000100:00080000:0.0:1788880359.549744:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880359.549747:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880359.549750:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880359.549758:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511739520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880359.549766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880359.549819:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880359/real 1788880359] req@ffffa06198afdf80 x1875771511739008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880359.549837:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511739008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880359.549848:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880359.549852:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880359.549856:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511739008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880359.549860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880359.549876:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880359.549884:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880359/real 1788880359] req@ffffa06198afdc00 x1875771511739136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880359.549893:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511739136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880359.549899:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880359.549902:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880359.549904:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880359.549930:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880359.549940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880359.549946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880359/real 1788880359] req@ffffa06198afc700 x1875771511739264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880359.549959:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511739264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880359.549965:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880359.549967:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880359.549968:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880359.549972:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880359.549984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880359.549990:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880359/real 1788880359] req@ffffa06198afc000 x1875771511739392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880359.549997:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511739392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880359.550003:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880359.550004:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880359.550006:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880359.550009:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880359.550019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880359.550025:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880359/real 1788880359] req@ffffa06198afea00 x1875771511739520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880359.550033:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511739520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880359.550043:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880359.550045:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880359.550047:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880360.189506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880360.189546:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880360.445441:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880361.213469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880361.213566:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880361.469580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880362.237491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880362.237583:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880362.493468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880363.261557:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880363.261648:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880363.517514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880364.285491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880364.285593:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880364.541498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880364.669553:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880364.669562:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880364.669566:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880364.669577:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5351 00000100:00080000:0.0:1788880364.669588:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880364.669594:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880364.669695:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511739648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880364.669719:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880364.669722:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880364.669723:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880364.669727:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5351 00000100:00080000:0.0:1788880364.669730:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880364.669733:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880364.669743:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880364.669752:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511739776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880364.669759:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880364.669762:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880364.669763:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880364.669765:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5351 00000100:00080000:0.0:1788880364.669769:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880364.669771:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880364.669775:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880364.669782:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511739904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880364.669792:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880364.669795:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880364.669796:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880364.669801:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5351 00000100:00080000:0.0:1788880364.669804:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880364.669812:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880364.669813:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511739648:192.168.203.111@tcp:250:kworker.0 00000008:00100000:0.0:1788880364.669819:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880364.669833:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511740032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880364.669841:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880364.669844:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880364.669845:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880364.669849:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5351 00000100:00080000:0.0:1788880364.669852:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880364.669854:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880364.669860:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880364.669869:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511740160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880364.669894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880364.670039:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880364/real 1788880364] req@ffffa06198afd180 x1875771511739648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880364.670059:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511739648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880364.670070:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880364.670074:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880364.670080:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511739648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880364.670088:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880364.670107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880364.670116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880364/real 1788880364] req@ffffa06198afe300 x1875771511739776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880364.670129:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511739776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880364.670135:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880364.670137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880364.670139:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880364.670159:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880364.670171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880364.670179:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880364/real 1788880364] req@ffffa06198afc380 x1875771511739904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880364.670187:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511739904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880364.670193:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880364.670195:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880364.670197:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511739904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880364.670201:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880364.670213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880364.670220:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880364/real 1788880364] req@ffffa06198afca80 x1875771511740032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880364.670228:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511740032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880364.670238:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880364.670241:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880364.670243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880364.670247:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880364.670257:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880364.670264:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880364/real 1788880364] req@ffffa06198afd500 x1875771511740160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880364.670271:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511740160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880364.670278:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880364.670279:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880364.670281:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880365.309462:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880365.309546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880365.565521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880366.333471:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880366.333542:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880366.589478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880367.357485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880367.357643:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880367.613508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880368.381479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880368.381732:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880368.637510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880369.405492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880369.405637:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880369.661499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880369.789502:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880369.789513:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880369.789519:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880369.789534:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5356 00000100:00080000:0.0:1788880369.789550:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880369.789585:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880369.789645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511740288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880369.789685:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880369.789690:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880369.789693:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880369.789709:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5356 00000100:00080000:0.0:1788880369.789716:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880369.789722:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880369.789739:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880369.789758:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511740416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880369.789775:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511740288:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880369.789778:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880369.789793:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880369.789797:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880369.789803:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5356 00000100:00080000:0.0:1788880369.789807:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880369.789811:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880369.789818:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880369.789833:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511740544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880369.789847:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880369.789852:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880369.789855:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880369.789860:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5356 00000100:00080000:0.0:1788880369.789867:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880369.789872:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880369.789880:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880369.789895:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511740672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880369.789907:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880369.789912:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880369.789914:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880369.789920:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5356 00000100:00080000:0.0:1788880369.789925:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880369.789931:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880369.789938:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880369.789956:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511740800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880369.789956:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880369.790029:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880369.790078:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880369.790131:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880370.429482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880370.429650:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880370.685507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880371.453508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880371.453728:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880371.709508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880372.477569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880372.477625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880372.733505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880372.861814:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880372.861890:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880372.861916:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880372.862008:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880372.862029:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880372.862051:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880372.862073:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:2.0:1788880372.862089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880369/real 1788880372] req@ffffa06198afca80 x1875771511740416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880372.862141:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511740416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880372.862177:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880372.862204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880372.862213:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880372.862231:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880369/real 1788880372] req@ffffa06198afc380 x1875771511740544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880372.862257:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511740544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880372.862272:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880372.862279:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880372.862286:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880372.862301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880369/real 1788880372] req@ffffa06198afe300 x1875771511740672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880372.862353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511740672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880372.862370:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880372.862378:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880372.862437:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880372.862452:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880369/real 1788880372] req@ffffa06198afd500 x1875771511740288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880372.862480:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511740288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880372.862496:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880372.862514:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880372.862521:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511740288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880372.862597:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880369/real 1788880372] req@ffffa06198afd180 x1875771511740800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880372.862625:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511740800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880372.862636:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880372.862641:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880372.862645:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511740800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880373.501520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880373.501749:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880373.757505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880374.525523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880374.525705:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880374.781502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880374.909619:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880374.909636:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880374.909646:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880374.909667:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5361 00000100:00080000:3.0:1788880374.909691:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880374.909707:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880374.909781:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511740928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880374.909837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880374.909861:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880374.909866:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880374.909875:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5361 00000100:00080000:3.0:1788880374.909883:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880374.909893:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880374.909913:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880374.909943:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511741056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880374.909952:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511740928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880374.909985:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880374.909994:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880374.909999:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880374.910007:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5361 00000100:00080000:3.0:1788880374.910016:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880374.910024:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880374.910036:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880374.910060:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511741184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880374.910086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880374.910157:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880374/real 1788880374] req@ffffa06188873480 x1875771511740928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880374.910214:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511740928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880374.910233:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880374.910246:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880374.910249:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880374.910252:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880374.910264:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788880374.910272:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5361 00000100:00100000:2.0:1788880374.910284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511740928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880374.910289:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880374.910303:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880374.910315:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880374.910333:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880374.910349:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511741312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880374.910375:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880374.910444:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880374.910449:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880374.910462:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5361 00000800:00000100:2.0:1788880374.910464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880374.910481:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880374.910491:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880374.910505:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880374.910507:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880374/real 1788880374] req@ffffa06188870a80 x1875771511741056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880374.910568:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511741440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880374.910576:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511741056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880374.910617:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880374.910627:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880374.910638:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880374.910656:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880374.910704:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880374.910723:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880374/real 1788880374] req@ffffa06188872d80 x1875771511741184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880374.910741:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511741184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880374.910752:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880374.910755:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880374.910760:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880374.910782:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880374.910808:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880374.910821:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880374/real 1788880374] req@ffffa06188872300 x1875771511741312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880374.910835:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511741312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880374.910845:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880374.910849:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880374.910853:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880374.910860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880374.910879:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880374.910890:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880374/real 1788880374] req@ffffa06188873b80 x1875771511741440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880374.910903:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511741440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880374.910921:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880374.910925:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880374.910929:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880375.549513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880375.549681:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880375.805496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880376.573535:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880376.573655:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880376.829502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880377.597536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880377.597661:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880377.853496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880378.621476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880378.621539:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880378.877487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880379.645531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880379.645599:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880379.901479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880380.029470:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880380.029481:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880380.029486:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880380.029506:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5366 00000100:00080000:3.0:1788880380.029519:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880380.029527:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880380.029571:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511741568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880380.029595:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880380.029598:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880380.029599:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880380.029602:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5366 00000100:00080000:3.0:1788880380.029605:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880380.029607:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880380.029615:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880380.029626:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511741696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880380.029634:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880380.029636:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880380.029637:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880380.029640:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5366 00000100:00080000:3.0:1788880380.029642:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880380.029644:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880380.029647:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880380.029654:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511741568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880380.029662:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511741824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880380.029670:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880380.029673:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880380.029675:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880380.029678:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5366 00000100:00080000:3.0:1788880380.029681:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880380.029683:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880380.029686:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880380.029697:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511741952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880380.029703:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880380.029706:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880380.029707:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880380.029709:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5366 00000100:00080000:3.0:1788880380.029712:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880380.029715:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880380.029717:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1788880380.029725:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880380.029727:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511742080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880380.029766:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880380/real 1788880380] req@ffffa06188872a00 x1875771511741568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880380.029780:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511741568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880380.029789:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880380.029792:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880380.029795:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511741568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880380.029800:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880380.029814:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880380.029822:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880380/real 1788880380] req@ffffa06188871c00 x1875771511741696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880380.029834:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511741696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880380.029840:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880380.029842:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880380.029844:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880380.029866:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880380.029876:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880380.029883:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880380/real 1788880380] req@ffffa06188873800 x1875771511741824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880380.029890:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511741824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880380.029895:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880380.029897:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880380.029899:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880380.029902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880380.029912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880380.029917:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880380/real 1788880380] req@ffffa06188871880 x1875771511741952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880380.029924:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511741952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880380.029934:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880380.029936:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880380.029938:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511741952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880380.029941:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880380.029950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880380.029955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880380/real 1788880380] req@ffffa06188870380 x1875771511742080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880380.029962:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511742080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880380.029967:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880380.029969:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880380.029971:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880380.669472:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880380.669518:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880380.925455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880381.693473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880381.693556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880381.949461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880382.717520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880382.717606:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880382.973581:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880383.741547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880383.741618:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880383.997499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880384.765530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880384.765607:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880385.021527:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880385.149641:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880385.149656:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880385.149661:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880385.149673:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5371 00000100:00080000:0.0:1788880385.149688:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880385.149698:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880385.149761:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511742208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880385.149791:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880385.149794:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880385.149796:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880385.149799:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5371 00000100:00080000:0.0:1788880385.149803:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880385.149806:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880385.149818:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880385.149830:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511742336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880385.149840:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788880385.149842:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511742208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880385.149843:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880385.149847:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880385.149855:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5371 00000100:00080000:0.0:1788880385.149858:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880385.149860:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880385.149863:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880385.149871:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511742464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880385.149878:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880385.149880:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880385.149882:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880385.149884:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5371 00000100:00080000:0.0:1788880385.149886:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880385.149900:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880385.149903:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880385.149910:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511742592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880385.149916:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880385.149918:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880385.149919:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880385.149921:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5371 00000100:00080000:0.0:1788880385.149924:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880385.149926:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880385.149930:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880385.149937:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511742720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880385.149945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880385.149985:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880385/real 1788880385] req@ffffa06198afea00 x1875771511742208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880385.150004:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511742208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880385.150013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880385.150016:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880385.150019:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511742208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880385.150041:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880385.150058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880385.150066:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880385/real 1788880385] req@ffffa06198afc000 x1875771511742336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880385.150074:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511742336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880385.150080:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880385.150082:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880385.150084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880385.150087:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880385.150098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880385.150103:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880385/real 1788880385] req@ffffa06198afc700 x1875771511742464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880385.150111:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511742464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880385.150120:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880385.150122:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880385.150125:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880385.150128:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880385.150139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880385.150144:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880385/real 1788880385] req@ffffa06198afdc00 x1875771511742592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880385.150151:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511742592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880385.150157:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880385.150159:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880385.150160:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880385.150164:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880385.150181:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880385.150186:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880385/real 1788880385] req@ffffa06198afdf80 x1875771511742720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880385.150193:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511742720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880385.150197:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880385.150199:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880385.150202:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880385.789539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880385.789625:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880386.045547:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880386.813479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880386.813560:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880387.069586:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880387.837511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880387.837577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880388.093555:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880388.861492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880388.861580:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880389.117456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880389.885545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880389.885614:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880390.141471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880390.269544:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880390.269554:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880390.269558:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880390.269569:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5376 00000100:00080000:0.0:1788880390.269580:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880390.269587:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880390.269632:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511742848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880390.269669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880390.269672:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880390.269674:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880390.269677:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5376 00000100:00080000:0.0:1788880390.269679:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880390.269682:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880390.269693:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880390.269703:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511742976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880390.269710:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880390.269713:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880390.269715:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880390.269718:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5376 00000100:00080000:0.0:1788880390.269720:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880390.269723:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880390.269725:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880390.269732:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511743104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880390.269740:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788880390.269743:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511742848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880390.269744:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880390.269748:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880390.269754:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5376 00000100:00080000:0.0:1788880390.269756:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880390.269758:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880390.269762:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880390.269770:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511743232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880390.269775:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880390.269778:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880390.269779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880390.269782:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5376 00000100:00080000:0.0:1788880390.269785:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880390.269787:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880390.269790:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880390.269798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511743360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880390.269823:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880390.269870:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880390/real 1788880390] req@ffffa06198aff800 x1875771511742848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880390.269887:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511742848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880390.269896:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880390.269899:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880390.269903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511742848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880390.269908:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880390.269923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880390.269930:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880390/real 1788880390] req@ffffa06198affb80 x1875771511742976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880390.269938:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511742976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880390.269944:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880390.269947:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880390.269949:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511742976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880390.269972:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880390.269983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880390.269989:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880390/real 1788880390] req@ffffa06198afd880 x1875771511743104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880390.269997:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511743104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880390.270002:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880390.270004:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880390.270006:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880390.270010:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880390.270020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880390.270025:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880390/real 1788880390] req@ffffa06198aff480 x1875771511743232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880390.270032:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511743232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880390.270038:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880390.270039:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880390.270041:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880390.270044:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880390.270065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880390.270072:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880390/real 1788880390] req@ffffa06198afed80 x1875771511743360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880390.270079:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511743360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880390.270084:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880390.270087:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880390.270089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880390.909509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880390.909571:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880391.165464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880391.933515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880391.933611:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880392.189553:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880392.957469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880392.957520:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880393.213475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880393.981508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880393.981592:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880394.237476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880395.005481:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880395.005562:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880395.261461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880395.389584:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880395.389594:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880395.389601:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880395.389613:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5382 00000100:00080000:1.0:1788880395.389625:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880395.389634:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880395.389678:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511743488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880395.389707:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880395.389711:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880395.389713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880395.389717:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5382 00000100:00080000:1.0:1788880395.389720:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880395.389732:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880395.389744:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880395.389758:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511743616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880395.389766:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511743488:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880395.389780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880395.389784:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880395.389785:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880395.389789:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5382 00000100:00080000:1.0:1788880395.389792:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880395.389795:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880395.389800:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880395.389810:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511743744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880395.389819:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880395.389823:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880395.389824:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880395.389827:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5382 00000100:00080000:1.0:1788880395.389830:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880395.389833:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880395.389839:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880395.389869:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511743872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788880395.389878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880395.389896:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880395.389901:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880395.389904:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880395.389911:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5382 00000100:00080000:1.0:1788880395.389916:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880395.389922:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880395.389929:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880395.389950:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880395/real 1788880395] req@ffffa0618869b100 x1875771511743488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880395.389952:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511744000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880395.390038:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511743488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880395.390055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880395.390062:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880395.390068:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511743488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880395.390102:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880395.390133:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880395.390148:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880395/real 1788880395] req@ffffa06188698380 x1875771511743616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880395.390164:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511743616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880395.390174:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880395.390178:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880395.390183:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880395.390190:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880395.390229:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880395.390243:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880395/real 1788880395] req@ffffa0618869b800 x1875771511743744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880395.390267:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511743744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880395.390283:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880395.390290:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880395.390296:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880395.390308:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880395.390349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880395.390424:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880395/real 1788880395] req@ffffa06188699c00 x1875771511743872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880395.390438:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511743872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880395.390445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880395.390448:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880395.390451:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511743872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880395.390475:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880395.390506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880395.390520:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880395/real 1788880395] req@ffffa0618869aa00 x1875771511744000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880395.390534:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511744000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880395.390545:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880395.390549:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880395.390553:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744000:192.168.203.111@tcp:8:kworker.0 00010000:00010000:0.0:1788880396.029510:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880396.029510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788880396.285450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880397.053526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880397.053575:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880397.309467:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880398.077511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880398.077571:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880398.333529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880399.101540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880399.101570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880399.357464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880400.125468:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880400.125573:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880400.381496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880400.509493:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880400.509504:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880400.509509:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880400.509523:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5387 00000100:00080000:2.0:1788880400.509536:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880400.509545:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880400.509590:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511744128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880400.509617:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880400.509620:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880400.509621:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880400.509624:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5387 00000100:00080000:2.0:1788880400.509627:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880400.509631:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880400.509642:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880400.509655:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511744256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880400.509667:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511744128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880400.509669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880400.509679:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880400.509685:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880400.509690:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5387 00000100:00080000:2.0:1788880400.509692:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880400.509694:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880400.509698:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880400.509710:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511744384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880400.509717:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880400.509720:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880400.509721:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880400.509724:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5387 00000100:00080000:2.0:1788880400.509727:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880400.509730:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880400.509734:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880400.509741:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511744512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880400.509749:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880400.509755:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788880400.509758:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880400.509759:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880400.509764:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5387 00000100:00080000:2.0:1788880400.509769:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880400.509772:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880400.509776:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880400.509786:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511744640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880400.509801:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880400/real 1788880400] req@ffffa061888af100 x1875771511744128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880400.509823:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511744128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880400.509832:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880400.509835:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880400.509838:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511744128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880400.509860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880400.509876:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880400.509885:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880400/real 1788880400] req@ffffa061888afb80 x1875771511744256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880400.509892:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511744256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880400.509898:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880400.509900:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880400.509902:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880400.509906:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880400.509919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880400.509925:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880400/real 1788880400] req@ffffa061888af480 x1875771511744384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880400.509933:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511744384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880400.509937:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880400.509939:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880400.509941:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880400.509945:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880400.509955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880400.509961:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880400/real 1788880400] req@ffffa061888adf80 x1875771511744512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880400.509968:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511744512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880400.509973:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880400.509975:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880400.509977:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880400.509981:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880400.509990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880400.509996:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880400/real 1788880400] req@ffffa061888aca80 x1875771511744640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880400.510007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511744640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880400.510012:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880400.510014:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880400.510016:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880401.149543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880401.149631:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880401.405517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880402.173489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880402.173572:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880402.429520:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880403.197529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880403.197653:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880403.453533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880404.221474:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880404.221538:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880404.477450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880405.245460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880405.245608:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880405.501488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880405.629534:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880405.629546:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880405.629554:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880405.629572:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5392 00000100:00080000:2.0:1788880405.629590:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880405.629603:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880405.629679:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511744768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880405.629726:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880405.629733:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880405.629737:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880405.629746:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5392 00000100:00080000:2.0:1788880405.629754:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880405.629764:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880405.629786:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880405.629795:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511744768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880405.629824:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511744896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880405.629850:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880405.629856:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880405.629861:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880405.629868:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5392 00000100:00080000:2.0:1788880405.629875:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880405.629881:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880405.629894:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788880405.629906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880405.629923:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511745024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880405.629941:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880405.629945:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880405.629948:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880405.629955:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5392 00000100:00080000:2.0:1788880405.629961:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880405.629968:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880405/real 1788880405] req@ffffa061888ac000 x1875771511744768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880405.629973:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880405.629994:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880405.630024:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511745152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880405.630027:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511744768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880405.630113:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880405.630120:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880405.630121:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880405.630134:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880405.630139:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:2.0:1788880405.630146:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5392 00000100:00100000:0.0:1788880405.630157:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511744768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880405.630158:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880405.630174:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880405.630185:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880405.630205:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511745280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880405.630212:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880405.630277:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880405.630334:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880405/real 1788880405] req@ffffa061888ad500 x1875771511744896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880405.630365:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511744896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880405.630425:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880405.630432:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880405.630437:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511744896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880405.630447:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880405.630481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880405.630498:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880405/real 1788880405] req@ffffa061888aca80 x1875771511745024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880405.630514:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511745024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880405.630524:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880405.630528:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880405.630532:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880405.630540:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880405.630561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880405.630573:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880405/real 1788880405] req@ffffa061888adf80 x1875771511745152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880405.630587:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511745152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880405.630607:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880405.630611:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880405.630615:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880405.630639:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880405.630659:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880405.630672:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880405/real 1788880405] req@ffffa061888af480 x1875771511745280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880405.630686:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511745280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880405.630696:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880405.630701:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880405.630704:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880406.269456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880406.269515:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880406.525529:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880407.293540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880407.293676:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880407.549457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880408.317476:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880408.317551:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880408.573512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880409.341460:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880409.341592:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880409.597468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880410.365454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880410.365552:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880410.621432:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880410.749528:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880410.749539:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880410.749545:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880410.749572:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5397 00000100:00080000:2.0:1788880410.749583:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880410.749590:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880410.749637:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511745408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880410.749661:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880410.749664:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880410.749665:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880410.749668:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5397 00000100:00080000:2.0:1788880410.749672:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880410.749675:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880410.749687:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880410.749698:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511745536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880410.749706:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511745408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880410.749720:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880410.749724:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880410.749725:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880410.749728:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5397 00000100:00080000:2.0:1788880410.749732:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880410.749735:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880410.749740:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880410.749762:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511745664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880410.749769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880410.749772:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880410.749773:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880410.749777:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5397 00000100:00080000:2.0:1788880410.749781:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880410.749785:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880410.749791:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880410.749798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511745792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880410.749805:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880410.749808:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880410.749810:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880410.749842:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880410.749850:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5397 00000100:00080000:2.0:1788880410.749860:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880410.749870:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880410.749877:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880410.749899:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511745920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880410.749916:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880410/real 1788880410] req@ffffa061888afb80 x1875771511745408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880410.749948:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511745408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880410.749965:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880410.749970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880410.749977:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511745408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880410.750011:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880410.750039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880410.750054:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880410/real 1788880410] req@ffffa061888af100 x1875771511745536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880410.750070:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511745536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880410.750081:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880410.750085:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880410.750089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880410.750096:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880410.750117:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880410.750140:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880410/real 1788880410] req@ffffa061888ae300 x1875771511745664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880410.750148:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511745664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880410.750153:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880410.750155:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880410.750157:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880410.750163:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880410.750198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880410.750205:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880410/real 1788880410] req@ffffa061888aea00 x1875771511745792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880410.750213:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511745792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880410.750219:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880410.750222:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880410.750224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880410.750227:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880410.750237:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880410.750243:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880410/real 1788880410] req@ffffa061888adc00 x1875771511745920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880410.750250:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511745920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880410.750256:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880410.750258:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880410.750260:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511745920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880411.389441:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880411.389629:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880411.645501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880412.413463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880412.413611:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880412.669521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880413.437520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880413.437639:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880413.693517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880414.461492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880414.461626:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880414.717496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880415.485520:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880415.485606:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880415.741486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880415.869598:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880415.869609:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880415.869616:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880415.869633:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5402 00000100:00080000:1.0:1788880415.869649:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880415.869660:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880415.869717:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511746048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880415.869758:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880415.869764:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880415.869767:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880415.869774:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5402 00000100:00080000:1.0:1788880415.869780:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880415.869787:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880415.869803:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880415.869816:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511746048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880415.869825:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511746176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880415.869843:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880415.869847:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880415.869849:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880415.869854:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5402 00000100:00080000:1.0:1788880415.869858:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880415.869863:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880415.869869:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880415.869886:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511746304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880415.869899:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880415.869904:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880415.869907:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880415.869911:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5402 00000100:00080000:1.0:1788880415.869916:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880415.869924:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880415.869926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:1.0:1788880415.869933:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880415.869950:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511746432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880415.869964:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880415.869969:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880415.869972:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880415.869977:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5402 00000100:00080000:1.0:1788880415.869982:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880415.869985:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880415/real 1788880415] req@ffffa0618869aa00 x1875771511746048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880415.869991:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880415.870004:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880415.870027:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511746560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880415.870035:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511746048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880415.870072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880415.870078:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880415.870085:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511746048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880415.870119:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880415.870147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880415.870162:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880415/real 1788880415] req@ffffa06188699c00 x1875771511746176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880415.870177:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511746176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880415.870188:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880415.870192:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880415.870196:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880415.870203:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880415.870221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880415.870232:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880415/real 1788880415] req@ffffa0618869b800 x1875771511746304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880415.870289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511746304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880415.870307:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880415.870314:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880415.870321:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880415.870335:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880415.870378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880415.870427:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880415/real 1788880415] req@ffffa06188698380 x1875771511746432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880415.870443:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511746432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880415.870454:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880415.870458:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880415.870463:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880415.870470:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880415.870495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880415.870506:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880415/real 1788880415] req@ffffa0618869b100 x1875771511746560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880415.870520:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511746560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880415.870539:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880415.870543:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880415.870547:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880416.509502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880416.509635:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880416.765483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880417.533523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880417.533592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880417.789508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880418.557510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880418.557625:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880418.813470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880419.581453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880419.581512:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880419.837439:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880420.605511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880420.605604:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880420.861474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880420.989535:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880420.989548:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880420.989557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880420.989578:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5407 00000100:00080000:2.0:1788880420.989596:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880420.989612:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880420.989679:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511746688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880420.989724:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880420.989730:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880420.989733:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880420.989739:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5407 00000100:00080000:2.0:1788880420.989746:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880420.989752:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880420.989769:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880420.989791:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511746688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880420.989792:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511746816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880420.989820:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880420.989824:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880420.989826:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880420.989831:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5407 00000100:00080000:2.0:1788880420.989836:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880420.989840:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880420.989847:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880420.989864:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511746944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880420.989887:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880420.989891:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880420.989894:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880420.989898:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5407 00000100:00080000:2.0:1788880420.989903:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880420.989911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880420.989916:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880420.989925:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880420.989940:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511747072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880420.989952:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880420.989957:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880420.989960:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880420.989967:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5407 00000100:00100000:0.0:1788880420.989974:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880420/real 1788880420] req@ffffa061888aed80 x1875771511746688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880420.989975:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880420.989989:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880420.990002:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880420.990022:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511747200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880420.990027:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511746688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880420.990066:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880420.990073:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880420.990079:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511746688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880420.990115:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880420.990142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880420.990157:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880420/real 1788880420] req@ffffa061888ac700 x1875771511746816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880420.990173:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511746816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880420.990185:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880420.990189:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880420.990193:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880420.990225:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880420.990249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880420.990263:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880420/real 1788880420] req@ffffa061888ace00 x1875771511746944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880420.990277:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511746944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880420.990287:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880420.990291:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880420.990295:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511746944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880420.990301:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880420.990321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880420.990333:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880420/real 1788880420] req@ffffa061888ac380 x1875771511747072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880420.990346:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511747072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880420.990356:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880420.990359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880420.990364:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880420.990371:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880420.990424:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880420.990438:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880420/real 1788880420] req@ffffa061888ad880 x1875771511747200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880420.990453:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511747200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880420.990463:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880420.990467:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880420.990471:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880421.629514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880421.629611:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880421.885480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880422.653510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880422.653605:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880422.909490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880423.677517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880423.677612:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880423.933486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880424.701499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880424.701574:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880424.957499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880425.725449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880425.725544:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880425.981462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880426.109484:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880426.109495:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880426.109500:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880426.109511:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5412 00000100:00080000:2.0:1788880426.109523:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880426.109531:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880426.109573:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511747328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880426.109595:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880426.109599:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880426.109600:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880426.109604:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5412 00000100:00080000:2.0:1788880426.109607:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880426.109611:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880426.109621:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880426.109631:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511747456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880426.109640:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511747328:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880426.109642:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880426.109648:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880426.109652:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880426.109656:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5412 00000100:00080000:2.0:1788880426.109658:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880426.109661:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880426.109664:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880426.109675:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511747584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880426.109681:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880426.109684:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880426.109685:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880426.109688:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5412 00000100:00080000:2.0:1788880426.109690:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880426.109693:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880426.109696:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880426.109703:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511747712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880426.109709:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880426.109712:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880426.109713:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880426.109718:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5412 00000800:00000100:0.0:1788880426.109719:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880426.109721:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880426.109725:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880426.109728:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880426.109741:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511747840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880426.109762:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880426/real 1788880426] req@ffffa061888ae680 x1875771511747328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880426.109777:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511747328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880426.109786:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880426.109789:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880426.109793:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511747328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880426.109814:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880426.109830:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880426.109838:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880426/real 1788880426] req@ffffa061888ad180 x1875771511747456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880426.109846:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511747456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880426.109852:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880426.109854:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880426.109856:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880426.109860:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880426.109870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880426.109876:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880426/real 1788880426] req@ffffa061888ad880 x1875771511747584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880426.109883:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511747584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880426.109889:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880426.109891:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880426.109892:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880426.109896:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880426.109905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880426.109912:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880426/real 1788880426] req@ffffa061888ac380 x1875771511747712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880426.109920:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511747712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880426.109925:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880426.109927:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880426.109928:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880426.109932:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880426.109942:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880426.109948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880426/real 1788880426] req@ffffa061888ace00 x1875771511747840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880426.109956:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511747840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880426.109961:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880426.109963:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880426.109964:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511747840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880426.749482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880426.749546:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880427.005460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880427.773458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880427.773552:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880428.029462:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880428.797519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880428.797589:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880429.053516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880429.821482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880429.821559:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880430.077458:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880430.845570:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880430.845673:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880431.101532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880431.229483:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880431.229494:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880431.229498:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880431.229509:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5417 00000100:00080000:2.0:1788880431.229520:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880431.229528:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880431.229573:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511747968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880431.229594:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880431.229597:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880431.229598:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880431.229602:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5417 00000100:00080000:2.0:1788880431.229605:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880431.229608:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880431.229617:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880431.229630:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511747968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880431.229630:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511748096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880431.229645:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880431.229647:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880431.229648:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880431.229651:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5417 00000100:00080000:2.0:1788880431.229653:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880431.229655:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880431.229658:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880431.229666:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511748224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880431.229674:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880431.229676:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880431.229678:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880431.229680:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5417 00000100:00080000:2.0:1788880431.229683:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880431.229685:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880431.229688:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880431.229695:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511748352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880431.229701:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788880431.229706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880431.229707:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880431.229712:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880431.229717:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5417 00000100:00080000:2.0:1788880431.229720:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880431.229723:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880431.229726:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880431.229737:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511748480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880431.229751:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880431/real 1788880431] req@ffffa061888ac700 x1875771511747968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880431.229767:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511747968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880431.229776:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880431.229780:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880431.229783:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511747968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880431.229803:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880431.229818:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880431.229825:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880431/real 1788880431] req@ffffa061888aed80 x1875771511748096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880431.229833:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511748096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880431.229838:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880431.229840:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880431.229842:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880431.229846:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880431.229856:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880431.229862:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880431/real 1788880431] req@ffffa061888adc00 x1875771511748224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880431.229869:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511748224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880431.229879:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880431.229881:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880431.229883:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880431.229887:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880431.229897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880431.229902:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880431/real 1788880431] req@ffffa061888aea00 x1875771511748352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880431.229909:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511748352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880431.229914:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880431.229916:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880431.229918:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880431.229921:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880431.229931:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880431.229937:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880431/real 1788880431] req@ffffa061888ae300 x1875771511748480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880431.229944:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511748480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880431.229949:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880431.229950:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880431.229953:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880431.869441:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880431.869565:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880432.125448:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880432.893442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880432.893548:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880433.149449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880433.917508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880433.917634:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880434.173515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880434.941501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880434.941671:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880435.197491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880435.965531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880435.965592:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880436.221446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880436.349525:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880436.349552:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880436.349557:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880436.349571:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5423 00000100:00080000:2.0:1788880436.349584:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880436.349593:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880436.349633:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511748608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880436.349658:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880436.349663:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880436.349666:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880436.349673:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5423 00000100:00080000:2.0:1788880436.349678:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880436.349685:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880436.349693:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511748608:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788880436.349700:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880436.349721:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511748736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880436.349748:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880436.349754:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880436.349758:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880436.349765:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5423 00000100:00080000:2.0:1788880436.349771:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880436.349777:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880436.349785:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880436.349803:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511748864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880436.349816:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880436.349821:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880436.349824:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880436.349829:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5423 00000100:00080000:2.0:1788880436.349835:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880436.349841:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880436.349848:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880436.349862:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511748992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880436.349872:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748736:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788880436.349884:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880436.349892:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880436.349896:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880436.349904:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5423 00000100:00080000:2.0:1788880436.349913:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880436.349922:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880436.349949:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880436.349953:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880436.349972:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511749120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880436.349985:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880436.350009:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880436.989530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880436.989657:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880437.245468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880438.013477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880438.013682:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880438.269502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880439.037546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880439.037623:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880439.293503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880439.421768:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880439.421839:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880439.421869:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880439.421940:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880439.421961:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880439.421980:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880439.422002:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788880439.422052:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880436/real 1788880439] req@ffffa061888aca80 x1875771511749120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880439.422131:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511749120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880439.422156:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880439.422166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880439.422174:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880439.422191:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880436/real 1788880439] req@ffffa061888adf80 x1875771511748992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880439.422221:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511748992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880439.422233:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880439.422237:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880439.422241:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880439.422249:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880436/real 1788880439] req@ffffa061888afb80 x1875771511748736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880439.422264:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511748736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880439.422280:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880439.422287:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880439.422293:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880439.422306:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880436/real 1788880439] req@ffffa061888af480 x1875771511748864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880439.422330:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511748864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880439.422345:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880439.422351:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880439.422357:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511748864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880439.422380:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880436/real 1788880439] req@ffffa061888af100 x1875771511748608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880439.422447:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511748608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880439.422463:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880439.422472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880439.422481:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511748608:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880440.061493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880440.061579:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880440.317501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880441.085508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880441.085622:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880441.341486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880441.469529:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880441.469540:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880441.469546:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880441.469563:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5428 00000100:00080000:2.0:1788880441.469580:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880441.469593:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880441.469645:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511749248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880441.469691:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880441.469696:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880441.469699:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880441.469704:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5428 00000100:00080000:2.0:1788880441.469708:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880441.469713:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880441.469727:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880441.469743:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511749376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880441.469756:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880441.469762:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880441.469764:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880441.469769:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5428 00000100:00080000:2.0:1788880441.469773:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880441.469783:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880441.469786:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511749248:192.168.203.111@tcp:250:kworker.0 00000008:00100000:2.0:1788880441.469789:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880441.469817:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511749504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880441.469841:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880441.469847:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880441.469852:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880441.469859:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5428 00000100:00080000:2.0:1788880441.469869:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880441.469878:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880441.469890:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880441.469917:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880441.469934:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511749632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880441.469958:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880441.469966:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880441.469970:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880441.469980:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5428 00000100:00080000:2.0:1788880441.469990:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880441.470002:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880441/real 1788880441] req@ffffa061888ad500 x1875771511749248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880441.470004:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880441.470020:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880441.470049:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511749760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880441.470091:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511749248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880441.470120:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880441.470131:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880441.470141:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511749248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880441.470157:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749376:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880441.470202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880441.470221:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880441/real 1788880441] req@ffffa061888ac000 x1875771511749376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880441.470239:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511749376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880441.470251:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880441.470256:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880441.470260:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880441.470291:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749504:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880441.470313:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880441.470326:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880441/real 1788880441] req@ffffa061888af100 x1875771511749504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880441.470340:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511749504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880441.470350:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880441.470354:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880441.470358:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880441.470365:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749632:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880441.470419:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880441.470441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880441/real 1788880441] req@ffffa061888af480 x1875771511749632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880441.470458:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511749632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880441.470469:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880441.470472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880441.470477:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880441.470486:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749760:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880441.470512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880441.470525:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880441/real 1788880441] req@ffffa061888afb80 x1875771511749760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880441.470539:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511749760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880441.470549:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880441.470552:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880441.470556:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511749760:192.168.203.111@tcp:8:kworker.0 00010000:00010000:2.0:1788880442.109498:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880442.110430:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788880442.365522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880443.133434:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880443.133500:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880443.389446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880444.157443:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880444.157488:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880444.413526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880445.181430:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880445.181502:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880445.437471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880446.205541:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880446.205639:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880446.461567:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880446.589546:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880446.589557:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880446.589562:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880446.589573:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5433 00000100:00080000:1.0:1788880446.589586:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880446.589593:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880446.589641:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511749888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880446.589666:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880446.589669:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880446.589670:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880446.589673:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5433 00000100:00080000:1.0:1788880446.589676:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880446.589679:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880446.589690:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880446.589701:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511749888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880446.589702:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511750016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880446.589715:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880446.589718:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880446.589719:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880446.589722:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5433 00000100:00080000:1.0:1788880446.589724:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880446.589726:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880446.589731:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880446.589739:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511750144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880446.589745:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880446.589748:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880446.589749:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880446.589752:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5433 00000100:00080000:1.0:1788880446.589766:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880446.589769:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880446.589772:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880446.589790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880446.589794:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511750272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880446.589801:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880446.589803:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880446.589805:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880446.589808:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5433 00000100:00080000:1.0:1788880446.589810:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880446.589813:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880446.589815:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880446.589822:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511750400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880446.589833:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880446/real 1788880446] req@ffffa06188699f80 x1875771511749888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880446.589849:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511749888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880446.589858:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880446.589862:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880446.589865:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511749888:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880446.589885:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750016:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880446.589901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880446.589909:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880446/real 1788880446] req@ffffa06188698000 x1875771511750016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880446.589917:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511750016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880446.589923:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880446.589925:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880446.589927:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880446.589931:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880446.589941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880446.589946:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880446/real 1788880446] req@ffffa0618869a680 x1875771511750144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880446.589954:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511750144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880446.589959:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880446.589961:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880446.589963:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880446.589966:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880446.589976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880446.589982:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880446/real 1788880446] req@ffffa0618869bb80 x1875771511750272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880446.589995:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511750272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880446.590000:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880446.590002:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880446.590003:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880446.590007:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880446.590016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880446.590041:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880446/real 1788880446] req@ffffa0618869ad80 x1875771511750400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880446.590049:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511750400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880446.590055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880446.590057:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880446.590059:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:1.0:1788880447.229465:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880447.229558:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880447.485471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880448.253484:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880448.253515:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880448.509450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880449.277456:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880449.277490:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880449.533471:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:1.0:1788880450.301452:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880450.301504:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880450.557484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880451.325568:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880451.325573:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880451.581476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880451.709539:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880451.709553:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880451.709559:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880451.709571:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5438 00000100:00080000:1.0:1788880451.709585:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880451.709595:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880451.709652:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511750528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880451.709680:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880451.709684:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880451.709686:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880451.709691:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5438 00000100:00080000:1.0:1788880451.709695:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880451.709700:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880451.709712:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880451.709721:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511750528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:1.0:1788880451.709727:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511750656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880451.709736:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880451.709740:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880451.709741:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880451.709745:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5438 00000100:00080000:1.0:1788880451.709749:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880451.709753:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880451.709758:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880451.709766:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511750784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880451.709775:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880451.709778:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880451.709779:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880451.709783:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5438 00000100:00080000:1.0:1788880451.709786:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880451.709789:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880451.709793:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880451.709802:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511750912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880451.709808:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880451.709811:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880451.709813:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880451.709815:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880451.709818:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5438 00000100:00080000:1.0:1788880451.709821:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880451.709825:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880451.709831:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880451.709842:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511751040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880451.709853:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880451/real 1788880451] req@ffffa0618869ad80 x1875771511750528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880451.709874:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511750528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880451.709887:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880451.709891:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880451.709895:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511750528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880451.709919:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880451.709939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880451.709948:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880451/real 1788880451] req@ffffa0618869bb80 x1875771511750656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880451.709956:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511750656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880451.709962:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880451.709965:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880451.709967:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880451.709970:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880451.709981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880451.710006:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880451/real 1788880451] req@ffffa0618869a680 x1875771511750784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880451.710015:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511750784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880451.710021:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880451.710023:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880451.710026:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880451.710030:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880451.710045:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880451.710052:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880451/real 1788880451] req@ffffa06188698000 x1875771511750912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880451.710059:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511750912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880451.710065:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880451.710067:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880451.710069:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511750912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880451.710072:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880451.710083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880451.710088:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880451/real 1788880451] req@ffffa06188699f80 x1875771511751040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880451.710095:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511751040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880451.710100:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880451.710102:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880451.710104:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880452.349448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880452.349536:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880452.605580:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880453.373449:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880453.373537:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880453.629476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880454.397511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880454.397533:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880454.653490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880455.421525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880455.421586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880455.677495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880456.445453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880456.445528:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880456.701443:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880456.829492:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880456.829498:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880456.829520:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880456.829534:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5443 00000100:00080000:3.0:1788880456.829551:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880456.829562:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880456.829619:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511751168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880456.829662:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880456.829667:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880456.829670:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880456.829676:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5443 00000100:00080000:3.0:1788880456.829681:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880456.829686:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880456.829702:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880456.829733:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511751168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880456.829734:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511751296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880456.829748:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880456.829751:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880456.829752:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880456.829754:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5443 00000100:00080000:3.0:1788880456.829757:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880456.829760:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880456.829764:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880456.829772:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511751424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880456.829779:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880456.829782:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880456.829784:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880456.829804:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5443 00000100:00080000:3.0:1788880456.829809:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880456.829813:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880456.829820:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880456.829840:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511751552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880456.829872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880456.829873:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880456.829879:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880456.829881:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880456.829886:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5443 00000100:00080000:3.0:1788880456.829891:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880456.829896:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880456.829902:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880456.829916:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880456/real 1788880456] req@ffffa06188870380 x1875771511751168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880456.829918:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511751680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880456.829959:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511751168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880456.829970:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880456.829974:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880456.829978:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511751168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880456.830011:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880456.830031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880456.830040:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880456/real 1788880456] req@ffffa06188871880 x1875771511751296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880456.830049:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511751296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880456.830055:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880456.830058:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880456.830060:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880456.830063:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880456.830075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880456.830081:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880456/real 1788880456] req@ffffa06188873800 x1875771511751424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880456.830105:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511751424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880456.830110:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880456.830112:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880456.830113:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880456.830117:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880456.830128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880456.830133:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880456/real 1788880456] req@ffffa06188871c00 x1875771511751552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880456.830140:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511751552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880456.830145:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880456.830147:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880456.830149:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880456.830152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880456.830162:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880456.830168:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880456/real 1788880456] req@ffffa06188872a00 x1875771511751680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880456.830174:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511751680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880456.830184:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880456.830187:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880456.830189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880457.469524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880457.469542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880457.725486:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880458.493440:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880458.493492:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880458.749446:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880459.517498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880459.517561:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880459.773442:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880460.541441:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880460.541463:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880460.797445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:3.0:1788880461.565646:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880461.565652:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880461.821510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880461.949543:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880461.949553:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880461.949560:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880461.949577:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5448 00000100:00080000:3.0:1788880461.949593:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880461.949604:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880461.949662:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511751808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880461.949698:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880461.949703:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880461.949706:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880461.949712:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5448 00000100:00080000:3.0:1788880461.949716:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880461.949722:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880461.949739:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880461.949757:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511751936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880461.949766:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511751808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880461.949785:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880461.949792:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880461.949794:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880461.949800:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5448 00000100:00080000:3.0:1788880461.949804:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880461.949808:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880461.949815:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880461.949833:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511752064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880461.949846:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880461.949851:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880461.949854:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880461.949859:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5448 00000100:00080000:3.0:1788880461.949865:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880461.949869:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880461.949876:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880461.949885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880461.949898:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511752192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880461.949967:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880461/real 1788880461] req@ffffa06188873b80 x1875771511751808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880461.949971:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880461.949989:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880461.949997:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880461.950007:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5448 00000100:00080000:3.0:1788880461.950013:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880461.950018:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511751808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880461.950026:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880461.950039:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880461.950050:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880461.950061:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880461.950067:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511752320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880461.950076:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511751808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880461.950133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880461.950183:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880461.950212:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880461/real 1788880461] req@ffffa06188872300 x1875771511751936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880461.950240:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511751936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880461.950256:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880461.950260:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880461.950267:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511751936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880461.950276:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880461.950305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880461.950321:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880461/real 1788880461] req@ffffa06188872d80 x1875771511752064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880461.950335:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511752064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880461.950346:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880461.950349:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880461.950354:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880461.950361:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880461.950413:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880461.950430:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880461/real 1788880461] req@ffffa06188870a80 x1875771511752192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880461.950445:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511752192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880461.950456:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880461.950460:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880461.950464:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880461.950471:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880461.950494:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880461.950507:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880461/real 1788880461] req@ffffa06188873480 x1875771511752320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880461.950521:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511752320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880461.950531:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880461.950534:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880461.950537:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880462.589553:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880462.589693:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880462.845511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788880463.613527:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880463.613534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788880463.869508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880464.637435:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880464.637482:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880464.893460:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880465.661511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880465.661534:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880465.917469:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880466.685442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880466.686460:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880466.941459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880467.069556:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880467.069566:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880467.069570:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880467.069580:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5453 00000100:00080000:1.0:1788880467.069591:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880467.069599:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880467.069640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511752448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880467.069667:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880467.069671:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880467.069672:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880467.069675:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5453 00000100:00080000:1.0:1788880467.069679:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880467.069682:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880467.069692:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880467.069701:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511752576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880467.069715:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511752448:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880467.069717:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880467.069736:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880467.069739:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880467.069743:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5453 00000100:00080000:1.0:1788880467.069745:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880467.069749:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880467.069753:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880467.069763:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511752704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880467.069770:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880467.069773:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880467.069775:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880467.069779:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5453 00000100:00080000:1.0:1788880467.069793:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880467.069796:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880467.069801:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880467.069805:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880467.069817:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511752832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880467.069833:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880467.069836:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880467.069837:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880467.069845:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5453 00000100:00100000:0.0:1788880467.069846:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880467/real 1788880467] req@ffffa0618869b100 x1875771511752448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880467.069852:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880467.069865:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880467.069872:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880467.069893:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511752960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880467.069901:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511752448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880467.069913:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880467.069918:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880467.069922:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511752448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880467.069944:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880467.069961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880467.069970:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880467/real 1788880467] req@ffffa06188698380 x1875771511752576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880467.069983:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511752576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880467.069989:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880467.069991:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880467.069993:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880467.069997:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880467.070008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880467.070014:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880467/real 1788880467] req@ffffa0618869b800 x1875771511752704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880467.070021:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511752704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880467.070026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880467.070028:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880467.070030:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880467.070033:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880467.070045:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880467.070051:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880467/real 1788880467] req@ffffa06188699c00 x1875771511752832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880467.070058:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511752832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880467.070068:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880467.070070:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880467.070072:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880467.070075:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880467.070085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880467.070090:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880467/real 1788880467] req@ffffa0618869aa00 x1875771511752960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880467.070097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511752960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880467.070102:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880467.070104:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880467.070106:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511752960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880467.709454:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880467.709515:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880467.965476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:1.0:1788880468.733496:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1788880468.734431:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:2.0:1788880468.989455:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880469.757495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880469.757555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880470.013517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880470.781546:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880470.781582:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880471.037498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880471.805531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880471.805696:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880472.061526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880472.189616:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880472.189628:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880472.189634:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880472.189647:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5458 00000100:00080000:3.0:1788880472.189662:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880472.189672:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880472.189720:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511753088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880472.189767:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880472.189772:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880472.189774:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880472.189780:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5458 00000100:00080000:3.0:1788880472.189784:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880472.189789:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880472.189803:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880472.189827:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511753088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880472.189828:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511753216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880472.189889:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880472.189894:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880472.189896:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880472.189900:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5458 00000100:00080000:3.0:1788880472.189905:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880472.189910:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880472.189917:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880472.189932:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511753344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880472.189939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880472.189958:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880472.189962:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880472.189963:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880472.189969:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5458 00000100:00080000:3.0:1788880472.189974:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880472.189977:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880472.189983:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880472.189984:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880472/real 1788880472] req@ffffa06188871500 x1875771511753088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880472.189997:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511753472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880472.190007:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511753088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880472.190009:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880472.190018:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880472.190021:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880472.190030:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5458 00000100:00080000:3.0:1788880472.190036:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880472.190043:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788880472.190045:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788880472.190047:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880472.190052:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880472.190056:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511753088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880472.190061:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511753600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880472.190082:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880472.190134:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880472.190152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880472/real 1788880472] req@ffffa06188870700 x1875771511753216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880472.190168:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511753216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880472.190178:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880472.190183:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880472.190196:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880472.190204:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880472.190225:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880472.190236:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880472/real 1788880472] req@ffffa06188872680 x1875771511753344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880472.190250:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511753344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880472.190260:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880472.190263:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880472.190267:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880472.190275:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880472.190294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880472.190305:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880472/real 1788880472] req@ffffa06188873100 x1875771511753472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880472.190319:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511753472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880472.190328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880472.190332:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880472.190335:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880472.190342:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880472.190359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880472.190369:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880472/real 1788880472] req@ffffa06188873480 x1875771511753600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880472.190382:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511753600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880472.190391:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880472.190394:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880472.190429:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880472.829477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880472.829520:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880473.085517:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880473.853509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880473.853583:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880474.109478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880474.877458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880474.877585:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880475.133507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880475.901509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880475.901566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880476.157497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880476.925511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880476.925567:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880477.181501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880477.309530:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880477.309541:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880477.309547:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880477.309563:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5464 00000100:00080000:3.0:1788880477.309580:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880477.309592:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880477.309641:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511753728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880477.309678:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880477.309683:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880477.309686:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880477.309691:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5464 00000100:00080000:3.0:1788880477.309696:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880477.309702:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880477.309716:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880477.309734:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511753856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880477.309742:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511753728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880477.309763:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880477.309769:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880477.309772:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880477.309778:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5464 00000100:00080000:3.0:1788880477.309783:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880477.309788:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880477.309795:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880477.309838:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511753984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880477.309852:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880477.309857:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880477.309860:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880477.309866:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5464 00000100:00080000:3.0:1788880477.309872:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880477.309878:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880477.309888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788880477.309893:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880477.309914:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511754112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880477.309926:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880477.309931:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880477.309934:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880477.309941:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5464 00000100:00080000:3.0:1788880477.309947:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880477.309950:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880477/real 1788880477] req@ffffa06188870a80 x1875771511753728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880477.309958:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880477.309972:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880477.309992:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511754240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880477.309997:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511753728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880477.310036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880477.310042:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880477.310049:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511753728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880477.310083:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880477.310114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880477.310130:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880477/real 1788880477] req@ffffa06188872d80 x1875771511753856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880477.310145:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511753856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880477.310156:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880477.310160:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880477.310164:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880477.310171:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880477.310191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880477.310203:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880477/real 1788880477] req@ffffa06188872300 x1875771511753984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880477.310217:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511753984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880477.310235:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880477.310239:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880477.310243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511753984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880477.310250:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880477.310269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880477.310280:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880477/real 1788880477] req@ffffa06188873b80 x1875771511754112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880477.310294:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511754112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880477.310304:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880477.310307:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880477.310311:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880477.310317:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880477.310336:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880477.310347:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880477/real 1788880477] req@ffffa06188872a00 x1875771511754240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880477.310361:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511754240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880477.310370:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880477.310374:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880477.310379:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880477.949510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880477.949668:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880478.205497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880478.973524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880478.973577:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880479.229550:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880479.997530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880479.997584:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880480.253508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880481.021517:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880481.021633:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880481.277511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880482.045511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880482.045561:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880482.301506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880482.429502:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880482.429513:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880482.429520:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880482.429536:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5469 00000100:00080000:3.0:1788880482.429553:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880482.429565:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880482.429615:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511754368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880482.429651:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880482.429656:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880482.429659:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880482.429665:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5469 00000100:00080000:3.0:1788880482.429669:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880482.429674:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880482.429687:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880482.429706:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511754496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880482.429714:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511754368:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880482.429738:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880482.429744:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880482.429747:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880482.429753:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5469 00000100:00080000:3.0:1788880482.429757:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880482.429761:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880482.429792:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880482.429813:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511754624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880482.429836:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880482.429841:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880482.429848:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880482.429849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880482.429859:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5469 00000100:00080000:3.0:1788880482.429867:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880482.429874:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880482.429882:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880482.429899:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511754752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880482.429913:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880482/real 1788880482] req@ffffa06188871c00 x1875771511754368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880482.429916:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880482.429930:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880482.429936:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880482.429952:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5469 00000100:00080000:3.0:1788880482.429958:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880482.429968:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880482.429969:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511754368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788880482.429974:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880482.429995:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880482.430004:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880482.430013:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511754880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880482.430016:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511754368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880482.430060:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880482.430093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880482.430109:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880482/real 1788880482] req@ffffa06188873800 x1875771511754496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880482.430126:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511754496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880482.430137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880482.430141:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880482.430145:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880482.430152:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880482.430171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880482.430185:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880482/real 1788880482] req@ffffa06188871880 x1875771511754624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880482.430199:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511754624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880482.430209:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880482.430213:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880482.430217:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880482.430224:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880482.430244:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880482.430255:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880482/real 1788880482] req@ffffa06188870380 x1875771511754752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880482.430270:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511754752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880482.430279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880482.430282:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880482.430286:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880482.430292:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880482.430310:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880482.430321:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880482/real 1788880482] req@ffffa06188872a00 x1875771511754880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880482.430335:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511754880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880482.430345:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880482.430349:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880482.430353:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511754880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880483.069501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880483.069567:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880483.325511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880484.093488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880484.093647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880484.349498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880485.117507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880485.117560:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880485.373512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880486.141516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880486.141571:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880486.397503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880487.165526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880487.165656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880487.421496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880487.549522:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880487.549533:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880487.549540:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880487.549555:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5474 00000100:00080000:3.0:1788880487.549572:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880487.549583:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880487.549629:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511755008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880487.549666:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880487.549672:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880487.549675:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880487.549680:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5474 00000100:00080000:3.0:1788880487.549685:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880487.549691:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880487.549704:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880487.549722:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511755008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880487.549724:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511755136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880487.549780:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880487.549785:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880487.549788:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880487.549794:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5474 00000100:00080000:3.0:1788880487.549800:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880487.549805:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880487.549814:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880487.549833:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511755264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880487.549854:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880487.549867:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880487.549873:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880487.549875:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880487.549883:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5474 00000100:00080000:3.0:1788880487.549888:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880487.549893:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880487.549901:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880487.549914:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880487/real 1788880487] req@ffffa06188873b80 x1875771511755008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880487.549921:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511755392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880487.549962:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880487.549967:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788880487.549974:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511755008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880487.549974:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880487.549989:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5474 00000100:00080000:3.0:1788880487.549999:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880487.550010:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788880487.550013:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788880487.550016:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880487.550025:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880487.550032:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511755008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880487.550040:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511755520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880487.550071:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880487.550102:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880487.550118:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880487/real 1788880487] req@ffffa06188872300 x1875771511755136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880487.550134:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511755136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880487.550145:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880487.550150:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880487.550154:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880487.550162:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880487.550182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880487.550194:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880487/real 1788880487] req@ffffa06188872d80 x1875771511755264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880487.550208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511755264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880487.550217:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880487.550221:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880487.550224:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880487.550231:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880487.550251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880487.550262:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880487/real 1788880487] req@ffffa06188870a80 x1875771511755392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880487.550275:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511755392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880487.550285:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880487.550289:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880487.550292:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880487.550299:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880487.550316:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880487.550327:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880487/real 1788880487] req@ffffa06188873480 x1875771511755520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880487.550341:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511755520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880487.550351:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880487.550354:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880487.550357:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880488.189505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880488.189566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880488.445496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880489.213527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880489.213608:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880489.469494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880490.237532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880490.237673:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880490.493519:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880491.261578:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880491.261619:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880491.517514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880492.285498:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880492.285559:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880492.541510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880492.669597:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880492.669612:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880492.669620:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880492.669639:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5479 00000100:00080000:2.0:1788880492.669658:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880492.669671:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880492.669750:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511755648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880492.669795:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880492.669801:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880492.669804:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880492.669810:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5479 00000100:00080000:2.0:1788880492.669815:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880492.669821:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880492.669836:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880492.669855:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511755776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880492.669871:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880492.669881:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788880492.669882:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511755648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880492.669884:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880492.669898:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5479 00000100:00080000:2.0:1788880492.669904:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880492.669909:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880492.669915:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880492.669932:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511755904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880492.669944:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880492.669949:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880492.669951:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880492.669957:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5479 00000100:00080000:2.0:1788880492.669962:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880492.669966:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880492.669972:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880492.669989:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511756032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880492.669997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880492.670014:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880492.670019:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880492.670021:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880492.670028:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5479 00000100:00080000:2.0:1788880492.670034:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880492.670039:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880492.670045:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880492.670063:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880492/real 1788880492] req@ffffa061888adf80 x1875771511755648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880492.670064:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511756160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880492.670143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511755648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880492.670160:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880492.670167:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880492.670172:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511755648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880492.670206:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880492.670233:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880492.670247:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880492/real 1788880492] req@ffffa061888aca80 x1875771511755776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880492.670263:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511755776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880492.670283:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880492.670287:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880492.670292:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880492.670299:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880492.670320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880492.670331:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880492/real 1788880492] req@ffffa061888ae300 x1875771511755904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880492.670345:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511755904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880492.670355:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880492.670358:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880492.670362:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511755904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880492.670369:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880492.670427:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880492.670443:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880492/real 1788880492] req@ffffa061888aea00 x1875771511756032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880492.670458:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511756032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880492.670468:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880492.670472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880492.670477:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880492.670485:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880492.670521:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880492.670535:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880492/real 1788880492] req@ffffa061888adc00 x1875771511756160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880492.670550:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511756160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880492.670561:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880492.670564:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880492.670568:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880493.309506:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880493.309666:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880493.565522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880494.333521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880494.333551:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880494.589464:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788880495.357554:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880495.357580:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788880495.613509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880496.381513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880496.381625:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880496.637480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880497.405521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880497.405598:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880497.661500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880497.789544:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880497.789555:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880497.789577:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880497.789587:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5484 00000100:00080000:0.0:1788880497.789598:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880497.789606:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880497.789650:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afed80 x1875771511756288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880497.789720:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880497.789725:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880497.789727:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880497.789734:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5484 00000100:00080000:0.0:1788880497.789738:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880497.789771:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880497.789784:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511756288:192.168.203.111@tcp:250:kworker.0 00000008:00100000:0.0:1788880497.789785:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880497.789803:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff480 x1875771511756416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880497.789813:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880497.789817:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880497.789818:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880497.789821:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5484 00000100:00080000:0.0:1788880497.789825:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880497.789828:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880497.789831:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880497.789841:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd880 x1875771511756544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880497.789848:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880497.789851:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880497.789852:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880497.789855:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5484 00000100:00080000:0.0:1788880497.789858:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880497.789860:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880497.789864:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880497.789871:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198affb80 x1875771511756672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880497.789877:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880497.789881:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880497.789883:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788880497.789897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788880497.789900:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5484 00000100:00080000:0.0:1788880497.789904:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880497.789907:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880497.789910:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880497.789917:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198aff800 x1875771511756800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788880497.789941:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880497/real 1788880497] req@ffffa06198afed80 x1875771511756288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880497.789960:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afed80 x1875771511756288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880497.789972:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880497.789976:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880497.789981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511756288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880497.790012:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880497.790035:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880497.790045:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880497/real 1788880497] req@ffffa06198aff480 x1875771511756416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880497.790054:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff480 x1875771511756416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880497.790061:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880497.790063:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880497.790066:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880497.790070:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:2.0:1788880497.790082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880497.790089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880497/real 1788880497] req@ffffa06198afd880 x1875771511756544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880497.790113:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd880 x1875771511756544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880497.790118:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880497.790120:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880497.790122:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880497.790126:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880497.790137:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880497.790143:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880497/real 1788880497] req@ffffa06198affb80 x1875771511756672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880497.790150:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198affb80 x1875771511756672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880497.790155:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880497.790157:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880497.790159:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198affb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880497.790162:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:2.0:1788880497.790181:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880497.790187:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880497/real 1788880497] req@ffffa06198aff800 x1875771511756800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880497.790199:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198aff800 x1875771511756800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880497.790205:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788880497.790207:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880497.790208:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198aff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511756800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880498.429521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880498.429551:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880498.685488:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880499.453505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880499.453699:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880499.709492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880500.477510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880500.477554:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880500.733495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880501.501469:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880501.501540:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880501.757511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880502.525523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880502.525687:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880502.781491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880502.909571:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880502.909580:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880502.909587:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880502.909601:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5489 00000100:00080000:2.0:1788880502.909615:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880502.909625:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880502.909703:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511756928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880502.909744:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880502.909750:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880502.909753:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880502.909758:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5489 00000100:00080000:2.0:1788880502.909763:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880502.909770:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880502.909787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880502.909808:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511756928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880502.909810:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511757056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880502.909835:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880502.909840:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880502.909842:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880502.909847:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5489 00000100:00080000:2.0:1788880502.909851:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880502.909855:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880502.909861:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880502.909879:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511757184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880502.909891:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880502.909896:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880502.909899:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880502.909904:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5489 00000100:00080000:2.0:1788880502.909908:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880502.909912:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880502.909919:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880502.909932:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511757312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880502.909945:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880502.909951:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880502.909954:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880502.909958:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5489 00000100:00080000:2.0:1788880502.909963:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880502.909972:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757056:192.168.203.111@tcp:38:kworker.0 00000100:00080000:2.0:1788880502.909973:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880502.909986:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880502.910007:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511757440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880502.910051:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880502.910087:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880502.910116:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757440:192.168.203.111@tcp:8:kworker.0 00010000:00010000:3.0:1788880503.549520:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880503.549537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788880503.805511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880504.573518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880504.573551:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880504.829505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880505.597519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880505.597609:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880505.853518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:2.0:1788880505.981886:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:2.0:1788880505.981974:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788880505.982000:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788880505.982059:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788880505.982072:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788880505.982083:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:2.0:1788880505.982097:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788880505.982139:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880502/real 1788880505] req@ffffa061888ad880 x1875771511757440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880505.982174:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511757440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880505.982190:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880505.982196:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880505.982202:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880505.982215:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880502/real 1788880505] req@ffffa061888ac700 x1875771511757056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880505.982231:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511757056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880505.982240:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880505.982244:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880505.982248:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880505.982256:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880502/real 1788880505] req@ffffa061888ace00 x1875771511757184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880505.982268:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511757184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880505.982277:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880505.982281:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880505.982285:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880505.982294:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880502/real 1788880505] req@ffffa061888ac380 x1875771511757312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880505.982307:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511757312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880505.982316:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880505.982319:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880505.982323:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880505.982331:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880502/real 1788880505] req@ffffa061888aed80 x1875771511756928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880505.982344:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511756928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880505.982352:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880505.982355:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880505.982360:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511756928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880506.621511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880506.621544:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880506.877511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880507.645514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880507.645564:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880507.901499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880508.029474:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880508.029484:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880508.029490:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880508.029505:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5494 00000100:00080000:2.0:1788880508.029519:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880508.029530:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880508.029590:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511757568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880508.029669:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880508.029679:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880508.029684:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880508.029695:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5494 00000100:00080000:2.0:1788880508.029706:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880508.029717:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880508.029747:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880508.029778:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511757696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880508.029796:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880508.029802:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880508.029805:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880508.029818:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5494 00000100:00080000:2.0:1788880508.029825:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788880508.029832:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511757568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880508.029835:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880508.029848:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880508.029867:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511757824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880508.029888:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880508.029892:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880508.029895:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880508.029900:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5494 00000100:00080000:2.0:1788880508.029905:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880508.029912:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880508.029923:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880508.029945:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511757952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880508.029965:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880508.029971:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880508.029975:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880508.029982:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5494 00000100:00080000:2.0:1788880508.029989:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880508.029997:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880508.030006:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880508.030029:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511758080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788880508.030147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880508.030213:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880508/real 1788880508] req@ffffa061888ad180 x1875771511757568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880508.030249:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511757568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880508.030271:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880508.030280:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880508.030288:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511757568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880508.030304:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757696:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880508.030351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880508.030378:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880508/real 1788880508] req@ffffa061888ae680 x1875771511757696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880508.030472:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511757696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880508.030490:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880508.030498:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880508.030506:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880508.030548:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757824:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880508.030622:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880508.030649:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880508/real 1788880508] req@ffffa061888aed80 x1875771511757824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880508.030667:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511757824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880508.030678:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880508.030682:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880508.030686:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880508.030695:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757952:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880508.030727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880508.030748:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880508/real 1788880508] req@ffffa061888ac380 x1875771511757952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880508.030772:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511757952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880508.030788:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880508.030796:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880508.030803:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511757952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880508.030816:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758080:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880508.030851:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880508.030868:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880508/real 1788880508] req@ffffa061888ace00 x1875771511758080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880508.030883:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511758080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880508.030894:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880508.030898:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880508.030902:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880508.669556:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880508.669658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880508.925507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880509.693569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880509.693569:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880509.949515:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880510.717502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880510.717601:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880510.973521:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880511.741507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880511.741609:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880511.997514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880512.765504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880512.765611:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880513.021503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880513.149539:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880513.149550:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880513.149556:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880513.149601:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5499 00000100:00080000:2.0:1788880513.149618:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880513.149631:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880513.149687:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511758208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880513.149723:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880513.149728:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880513.149731:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880513.149737:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5499 00000100:00080000:2.0:1788880513.149741:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880513.149745:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880513.149760:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880513.149776:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511758336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880513.149792:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880513.149797:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880513.149800:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788880513.149809:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511758208:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880513.149811:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5499 00000100:00080000:2.0:1788880513.149819:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880513.149828:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880513.149835:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880513.149853:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511758464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880513.149867:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880513.149871:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880513.149875:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880513.149879:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5499 00000100:00080000:2.0:1788880513.149898:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880513.149904:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880513.149911:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788880513.149920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880513.149930:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511758592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880513.149944:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880513.149949:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880513.149951:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880513.149956:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5499 00000100:00080000:2.0:1788880513.149962:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880513.149968:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880513.149975:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880513.149981:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880513/real 1788880513] req@ffffa061888ac700 x1875771511758208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880513.149996:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511758720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880513.150018:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511758208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880513.150036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880513.150043:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880513.150050:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511758208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880513.150082:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880513.150108:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880513.150124:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880513/real 1788880513] req@ffffa061888ad880 x1875771511758336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880513.150139:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511758336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880513.150150:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880513.150154:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880513.150159:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880513.150166:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880513.150185:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880513.150196:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880513/real 1788880513] req@ffffa061888adc00 x1875771511758464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880513.150211:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511758464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880513.150220:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880513.150224:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880513.150227:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880513.150234:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880513.150253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880513.150264:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880513/real 1788880513] req@ffffa061888aea00 x1875771511758592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880513.150278:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511758592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880513.150289:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880513.150293:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880513.150297:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880513.150303:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880513.150322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880513.150332:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880513/real 1788880513] req@ffffa061888ae300 x1875771511758720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880513.150345:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511758720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880513.150355:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880513.150359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880513.150363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880513.789514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880513.789538:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880514.045496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880514.813511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880514.813626:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880515.069490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880515.837477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880515.837598:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880516.093461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880516.861518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880516.861570:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880517.117483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880517.885564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880517.885663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880518.141560:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880518.269528:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880518.269561:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880518.269567:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880518.269582:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5504 00000100:00080000:0.0:1788880518.269596:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880518.269606:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880518.269660:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdf80 x1875771511758848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880518.269698:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880518.269703:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880518.269706:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880518.269711:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5504 00000100:00080000:0.0:1788880518.269716:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880518.269720:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880518.269735:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880518.269752:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afdc00 x1875771511758976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880518.269769:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880518.269777:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788880518.269780:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511758848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880518.269781:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880518.269792:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5504 00000100:00080000:0.0:1788880518.269799:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880518.269804:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880518.269810:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880518.269826:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc700 x1875771511759104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880518.269839:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880518.269844:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880518.269847:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880518.269851:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5504 00000100:00080000:0.0:1788880518.269855:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880518.269859:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880518.269865:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880518.269878:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc000 x1875771511759232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880518.269890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880518.269894:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880518.269896:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880518.269901:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5504 00000100:00080000:0.0:1788880518.269908:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788880518.269914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:0.0:1788880518.269918:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880518.269927:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880518.269943:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afea00 x1875771511759360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880518.269989:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880518/real 1788880518] req@ffffa06198afdf80 x1875771511758848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880518.270025:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdf80 x1875771511758848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880518.270051:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880518.270058:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880518.270065:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511758848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880518.270100:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880518.270133:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880518.270149:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880518/real 1788880518] req@ffffa06198afdc00 x1875771511758976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880518.270165:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afdc00 x1875771511758976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880518.270175:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880518.270180:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880518.270184:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511758976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880518.270191:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880518.270211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880518.270222:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880518/real 1788880518] req@ffffa06198afc700 x1875771511759104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880518.270236:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc700 x1875771511759104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880518.270247:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880518.270250:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880518.270254:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880518.270262:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880518.270295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880518.270316:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880518/real 1788880518] req@ffffa06198afc000 x1875771511759232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880518.270341:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc000 x1875771511759232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880518.270359:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880518.270366:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880518.270373:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880518.270431:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880518.270478:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880518.270502:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880518/real 1788880518] req@ffffa06198afea00 x1875771511759360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880518.270518:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afea00 x1875771511759360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880518.270529:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880518.270558:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880518.270562:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880518.909571:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880518.909595:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880519.165513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880519.933549:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880519.933601:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880520.189491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880520.957504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880520.957652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880521.213506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880521.981552:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880521.981588:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880522.237487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880523.005564:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880523.005597:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880523.261526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880523.389543:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880523.389554:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880523.389561:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880523.389575:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5510 00000100:00080000:0.0:1788880523.389589:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880523.389599:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880523.389655:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511759488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880523.389694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880523.389699:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880523.389702:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880523.389708:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5510 00000100:00080000:0.0:1788880523.389712:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880523.389716:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880523.389730:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880523.389746:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511759616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880523.389762:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880523.389771:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880523.389773:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880523.389778:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5510 00000100:00080000:0.0:1788880523.389782:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880523.389786:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880523.389793:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880523.389803:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511759488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880523.389813:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511759744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880523.389829:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880523.389832:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880523.389834:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880523.389840:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5510 00000100:00080000:0.0:1788880523.389844:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880523.389848:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880523.389854:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880523.389871:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511759872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880523.389883:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880523.389887:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880523.389890:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880523.389895:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5510 00000100:00080000:0.0:1788880523.389900:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880523.389904:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880523.389912:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788880523.389920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880523.389936:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511760000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880523.389981:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880523/real 1788880523] req@ffffa06198afd180 x1875771511759488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880523.390017:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511759488/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880523.390036:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880523.390044:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880523.390053:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511759488:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880523.390068:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759616:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880523.390099:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880523.390116:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880523/real 1788880523] req@ffffa06198afd500 x1875771511759616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880523.390131:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511759616/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880523.390142:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880523.390145:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880523.390150:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759616:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880523.390183:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759744:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880523.390205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880523.390217:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880523/real 1788880523] req@ffffa06198afe300 x1875771511759744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880523.390241:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511759744/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880523.390251:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880523.390254:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880523.390258:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759744:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880523.390265:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759872:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880523.390284:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880523.390296:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880523/real 1788880523] req@ffffa06198afc380 x1875771511759872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880523.390310:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511759872/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880523.390320:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880523.390323:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880523.390327:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511759872:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880523.390333:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760000:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880523.390352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880523.390362:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880523/real 1788880523] req@ffffa06198afca80 x1875771511760000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880523.390376:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511760000/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880523.390437:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880523.390441:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880523.390446:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760000:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880524.029547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880524.029641:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880524.285532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788880525.053566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880525.053567:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788880525.309535:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880526.077507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880526.077546:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880526.333549:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880527.101543:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880527.101623:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880527.357508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880528.125522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880528.125607:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880528.381495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:0.0:1788880528.509613:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880528.509623:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1788880528.509629:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880528.509643:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5515 00000100:00080000:0.0:1788880528.509657:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880528.509667:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880528.509721:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afca80 x1875771511760128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880528.509761:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880528.509766:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1788880528.509769:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880528.509774:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5515 00000100:00080000:0.0:1788880528.509779:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880528.509783:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880528.509797:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880528.509812:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afc380 x1875771511760256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880528.509825:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880528.509829:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1788880528.509832:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880528.509837:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5515 00000100:00080000:0.0:1788880528.509844:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880528.509849:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880528.509856:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880528.509871:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afe300 x1875771511760384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880528.509884:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788880528.509890:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511760128:192.168.203.111@tcp:250:kworker.0 00000100:00080000:0.0:1788880528.509891:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880528.509899:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880528.509913:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5515 00000100:00080000:0.0:1788880528.509917:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880528.509921:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880528.509927:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880528.509944:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd500 x1875771511760512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880528.509956:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788880528.509961:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1788880528.509963:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1788880528.509967:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5515 00000100:00080000:0.0:1788880528.509972:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1788880528.509976:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:0.0:1788880528.509982:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880528.509998:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06198afd180 x1875771511760640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788880528.510012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880528.510071:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880528/real 1788880528] req@ffffa06198afca80 x1875771511760128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880528.510097:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afca80 x1875771511760128/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880528.510112:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880528.510118:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880528.510124:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511760128:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880528.510133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760256:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880528.510159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880528.510174:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880528/real 1788880528] req@ffffa06198afc380 x1875771511760256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880528.510189:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afc380 x1875771511760256/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880528.510208:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880528.510212:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880528.510216:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760256:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880528.510248:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760384:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880528.510268:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880528.510281:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880528/real 1788880528] req@ffffa06198afe300 x1875771511760384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880528.510294:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afe300 x1875771511760384/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880528.510305:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880528.510309:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880528.510312:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afe300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760384:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880528.510319:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760512:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880528.510338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880528.510349:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880528/real 1788880528] req@ffffa06198afd500 x1875771511760512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880528.510363:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd500 x1875771511760512/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880528.510373:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880528.510376:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880528.510381:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760512:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880528.510429:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760640:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880528.510455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880528.510494:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880528/real 1788880528] req@ffffa06198afd180 x1875771511760640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880528.510511:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06198afd180 x1875771511760640/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880528.510522:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880528.510526:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880528.510530:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06198afd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760640:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880529.149531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880529.149585:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880529.405523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880530.173532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880530.173649:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880530.429567:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880531.197518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880531.197582:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880531.453552:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880532.221526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880532.221588:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880532.477513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880533.245559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880533.245644:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880533.501536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880533.629617:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880533.629627:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880533.629633:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880533.629649:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5520 00000100:00080000:2.0:1788880533.629664:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880533.629675:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880533.629728:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511760768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880533.629773:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880533.629781:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880533.629785:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880533.629793:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5520 00000100:00080000:2.0:1788880533.629801:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880533.629809:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880533.629829:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880533.629857:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511760896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880533.629890:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880533.629897:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1788880533.629902:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511760768:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880533.629903:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880533.629918:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5520 00000100:00080000:2.0:1788880533.629926:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880533.629932:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880533.629941:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880533.629961:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511761024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880533.629975:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880533.629980:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880533.629982:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880533.629986:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5520 00000100:00080000:2.0:1788880533.629991:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880533.629997:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880533.630003:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788880533.630015:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:2.0:1788880533.630023:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511761152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880533.630038:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880533.630044:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880533.630047:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880533.630052:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5520 00000100:00080000:2.0:1788880533.630059:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880533.630065:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880533.630072:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880533.630078:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880533/real 1788880533] req@ffffa061888aca80 x1875771511760768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788880533.630094:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511761280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880533.630116:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511760768/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880533.630136:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880533.630143:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880533.630150:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511760768:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880533.630185:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760896:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880533.630213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880533.630228:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880533/real 1788880533] req@ffffa061888adf80 x1875771511760896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880533.630244:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511760896/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880533.630255:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880533.630259:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880533.630264:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511760896:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880533.630271:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761024:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880533.630291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880533.630301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880533/real 1788880533] req@ffffa061888afb80 x1875771511761024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880533.630315:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511761024/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880533.630326:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880533.630329:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880533.630333:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761024:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880533.630340:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761152:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880533.630358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880533.630369:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880533/real 1788880533] req@ffffa061888af480 x1875771511761152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880533.630424:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511761152/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880533.630436:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880533.630466:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880533.630471:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761152:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880533.630480:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761280:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880533.630507:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880533.630522:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880533/real 1788880533] req@ffffa061888af100 x1875771511761280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880533.630537:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511761280/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880533.630548:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880533.630552:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880533.630556:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761280:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880534.269561:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880534.269570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880534.525544:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880535.293562:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880535.293600:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880535.549492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880536.317559:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880536.317633:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880536.573501:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880537.341524:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880537.341553:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880537.597468:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880538.365526:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880538.365558:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880538.621573:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880538.749479:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880538.749490:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880538.749495:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880538.749505:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5525 00000100:00080000:3.0:1788880538.749516:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880538.749523:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880538.749560:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511761408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880538.749584:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880538.749587:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880538.749588:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880538.749591:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5525 00000100:00080000:3.0:1788880538.749593:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880538.749597:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880538.749607:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880538.749620:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511761536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880538.749621:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511761408:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880538.749635:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880538.749638:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880538.749639:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880538.749641:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5525 00000100:00080000:3.0:1788880538.749643:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880538.749645:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880538.749648:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880538.749656:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511761664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880538.749663:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880538.749666:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880538.749667:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880538.749670:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5525 00000100:00080000:3.0:1788880538.749673:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880538.749675:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880538.749678:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880538.749687:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511761792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880538.749695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880538.749701:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880538.749705:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880538.749708:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880538.749713:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5525 00000100:00080000:3.0:1788880538.749718:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880538.749723:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880538.749729:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880538.749730:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880538/real 1788880538] req@ffffa06188873100 x1875771511761408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880538.749749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511761920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880538.749750:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511761408/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880538.749777:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880538.749781:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880538.749786:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511761408:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880538.749807:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761536:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880538.749821:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880538.749830:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880538/real 1788880538] req@ffffa06188872680 x1875771511761536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880538.749838:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511761536/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880538.749844:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880538.749846:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880538.749848:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761536:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880538.749852:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761664:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880538.749862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880538.749869:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880538/real 1788880538] req@ffffa06188870700 x1875771511761664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880538.749876:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511761664/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880538.749881:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880538.749883:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880538.749885:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761664:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880538.749888:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761792:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880538.749899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880538.749905:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880538/real 1788880538] req@ffffa06188871500 x1875771511761792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880538.749913:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511761792/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880538.749918:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880538.749920:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880538.749922:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761792:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880538.749925:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761920:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880538.749934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880538.749940:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880538/real 1788880538] req@ffffa06188873480 x1875771511761920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880538.749946:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511761920/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880538.749952:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880538.749954:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880538.749957:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511761920:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880539.389519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880539.389584:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880539.645476:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880540.413536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880540.413569:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880540.669594:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880541.437530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880541.437560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880541.693532:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880542.461458:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880542.461545:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880542.717499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880543.485540:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880543.485572:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880543.741571:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880543.869504:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880543.869517:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880543.869523:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880543.869535:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5530 00000100:00080000:3.0:1788880543.869549:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880543.869559:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880543.869612:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511762048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880543.869669:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880543.869677:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880543.869681:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880543.869690:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5530 00000100:00080000:3.0:1788880543.869698:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880543.869706:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880543.869725:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880543.869736:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511762048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880543.869750:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511762176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880543.869769:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880543.869775:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880543.869778:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880543.869785:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5530 00000100:00080000:3.0:1788880543.869792:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880543.869798:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880543.869807:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880543.869824:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511762304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880543.869840:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788880543.869844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880543.869847:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880543.869850:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880543.869856:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5530 00000100:00080000:3.0:1788880543.869862:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880543.869869:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880543.869876:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880543.869894:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511762432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880543.869908:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880543/real 1788880543] req@ffffa06188870a80 x1875771511762048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880543.869912:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880543.869919:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880543.869923:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880543.869929:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5530 00000100:00080000:3.0:1788880543.869935:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880543.869942:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880543.869943:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511762048/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788880543.869951:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880543.869964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788880543.869970:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511762560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880543.869973:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880543.869981:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511762048:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880543.870026:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762176:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880543.870070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880543.870089:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880543/real 1788880543] req@ffffa06188872d80 x1875771511762176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880543.870106:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511762176/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880543.870118:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880543.870122:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880543.870127:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762176:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880543.870135:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762304:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880543.870158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880543.870170:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880543/real 1788880543] req@ffffa06188872300 x1875771511762304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880543.870197:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511762304/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880543.870202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880543.870204:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880543.870206:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762304:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880543.870210:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762432:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880543.870221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880543.870227:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880543/real 1788880543] req@ffffa06188873b80 x1875771511762432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880543.870234:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511762432/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880543.870240:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880543.870241:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880543.870243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762432:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880543.870246:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762560:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880543.870256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880543.870262:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880543/real 1788880543] req@ffffa06188872a00 x1875771511762560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880543.870270:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511762560/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880543.870275:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880543.870277:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880543.870279:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762560:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880544.509499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880544.509522:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880544.765505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880545.533473:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880545.533561:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880545.789450:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880546.557455:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880546.557492:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880546.813445:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880547.581467:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880547.581489:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880547.837495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880548.605453:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880548.605540:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880548.861459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880548.989473:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880548.989481:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880548.989485:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880548.989497:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5535 00000100:00080000:3.0:1788880548.989510:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880548.989518:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880548.989576:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511762688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880548.989602:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880548.989605:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880548.989607:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880548.989610:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5535 00000100:00080000:3.0:1788880548.989614:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880548.989617:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880548.989645:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880548.989658:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511762688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880548.989658:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511762816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880548.989674:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880548.989677:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880548.989678:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880548.989681:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5535 00000100:00080000:3.0:1788880548.989683:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880548.989686:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880548.989690:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880548.989698:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511762944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880548.989704:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880548.989708:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880548.989709:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880548.989713:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5535 00000100:00080000:3.0:1788880548.989716:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880548.989718:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880548.989721:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880548.989729:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511763072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880548.989735:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880548.989739:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1788880548.989740:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880548.989741:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880548.989748:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5535 00000100:00080000:3.0:1788880548.989751:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880548.989754:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880548.989758:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880548.989768:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511763200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880548.989780:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880548/real 1788880548] req@ffffa06188870380 x1875771511762688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880548.989797:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511762688/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880548.989807:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880548.989811:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880548.989814:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511762688:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880548.989836:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762816:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880548.989852:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880548.989861:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880548/real 1788880548] req@ffffa06188871880 x1875771511762816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880548.989875:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511762816/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880548.989881:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880548.989883:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880548.989885:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762816:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880548.989889:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762944:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880548.989900:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880548.989907:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880548/real 1788880548] req@ffffa06188873800 x1875771511762944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880548.989915:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511762944/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880548.989921:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880548.989923:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880548.989924:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511762944:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880548.989928:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763072:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880548.989939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880548.989945:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880548/real 1788880548] req@ffffa06188871c00 x1875771511763072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880548.989953:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511763072/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880548.989964:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880548.989966:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880548.989968:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763072:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880548.989972:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763200:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880548.989981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880548.989987:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880548/real 1788880548] req@ffffa06188872a00 x1875771511763200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880548.989994:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511763200/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880548.990000:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880548.990002:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880548.990004:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763200:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880549.629479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880549.629522:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880549.885461:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880550.653497:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880550.653556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880550.909497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880551.677516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880551.677573:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880551.933537:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880552.701466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880552.701570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880552.957533:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880553.725508:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880553.725553:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880553.981479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880554.109517:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880554.109527:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880554.109535:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880554.109552:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5540 00000100:00080000:3.0:1788880554.109567:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880554.109579:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880554.109634:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511763328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880554.109672:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880554.109677:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880554.109680:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880554.109686:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5540 00000100:00080000:3.0:1788880554.109690:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880554.109696:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880554.109712:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880554.109737:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511763328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880554.109737:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511763456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880554.109767:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880554.109771:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880554.109773:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880554.109778:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5540 00000100:00080000:3.0:1788880554.109783:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880554.109787:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880554.109794:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880554.109813:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511763584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880554.109835:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880554.109840:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880554.109842:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880554.109847:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5540 00000100:00080000:3.0:1788880554.109851:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880554.109859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880554.109862:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880554.109874:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880554.109891:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511763712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880554.109904:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880554.109909:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880554.109912:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880554.109918:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5540 00000100:00080000:3.0:1788880554.109925:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880554.109926:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880554/real 1788880554] req@ffffa06188873b80 x1875771511763328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880554.109935:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880554.109948:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880554.109971:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511763328/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880554.109973:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511763840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880554.110033:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880554.110042:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880554.110050:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511763328:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880554.110083:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763456:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880554.110111:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880554.110127:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880554/real 1788880554] req@ffffa06188872300 x1875771511763456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880554.110143:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511763456/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880554.110154:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880554.110159:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880554.110163:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763456:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880554.110170:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763584:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880554.110190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880554.110201:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880554/real 1788880554] req@ffffa06188872d80 x1875771511763584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880554.110215:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511763584/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880554.110226:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880554.110230:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880554.110234:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763584:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880554.110243:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763712:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880554.110262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880554.110274:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880554/real 1788880554] req@ffffa06188870a80 x1875771511763712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880554.110288:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511763712/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880554.110299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880554.110302:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880554.110305:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763712:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880554.110339:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763840:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880554.110378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880554.110436:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880554/real 1788880554] req@ffffa06188873480 x1875771511763840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880554.110459:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511763840/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880554.110470:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880554.110475:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880554.110479:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511763840:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880554.749522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880554.749626:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880555.005484:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880555.773495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880555.773553:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880556.029475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880556.797493:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880556.797544:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880557.053478:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880557.821538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880557.821652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880558.077477:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880558.845515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880558.845566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880559.101482:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880559.229574:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880559.229598:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880559.229605:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880559.229620:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5545 00000100:00080000:1.0:1788880559.229634:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880559.229645:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880559.229701:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511763968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880559.229740:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880559.229745:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880559.229748:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880559.229753:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5545 00000100:00080000:1.0:1788880559.229758:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880559.229763:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880559.229778:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880559.229798:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511764096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880559.229803:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511763968:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880559.229839:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880559.229844:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880559.229847:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880559.229853:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5545 00000100:00080000:1.0:1788880559.229857:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880559.229861:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880559.229868:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880559.229886:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511764224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880559.229899:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880559.229904:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880559.229906:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880559.229910:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5545 00000100:00080000:1.0:1788880559.229916:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880559.229923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880559.229930:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880559.229939:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880559.229956:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511764352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880559.229971:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880559.229975:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880559.229978:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880559.229988:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5545 00000100:00100000:0.0:1788880559.229990:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880559/real 1788880559] req@ffffa0618869aa00 x1875771511763968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880559.229995:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880559.230008:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880559.230022:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880559.230041:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511764480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880559.230049:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511763968/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880559.230077:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880559.230083:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880559.230091:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511763968:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880559.230126:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764096:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880559.230154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880559.230169:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880559/real 1788880559] req@ffffa06188699c00 x1875771511764096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880559.230194:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511764096/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880559.230204:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880559.230209:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880559.230215:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764096:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880559.230226:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764224:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880559.230261:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880559.230312:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880559/real 1788880559] req@ffffa0618869b800 x1875771511764224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880559.230342:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511764224/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880559.230358:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880559.230362:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880559.230367:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764224:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880559.230376:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764352:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880559.230440:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880559.230456:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880559/real 1788880559] req@ffffa06188698380 x1875771511764352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880559.230472:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511764352/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880559.230482:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880559.230486:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880559.230490:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764352:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880559.230497:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764480:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880559.230516:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880559.230527:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880559/real 1788880559] req@ffffa0618869b100 x1875771511764480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880559.230547:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511764480/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880559.230566:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880559.230570:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880559.230574:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764480:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880559.869461:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880559.869479:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880560.125480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880560.893487:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880560.893583:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880561.149516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880561.917477:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880561.917491:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880562.173511:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880562.941516:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880562.941556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880563.197487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880563.965533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880563.965652:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880564.221513:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880564.349510:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880564.349520:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880564.349525:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880564.349539:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5551 00000100:00080000:1.0:1788880564.349557:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880564.349571:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880564.349640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511764608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880564.349695:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880564.349701:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880564.349705:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880564.349715:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5551 00000100:00080000:1.0:1788880564.349722:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880564.349729:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880564.349741:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511764608:192.168.203.111@tcp:250:kworker.0 00000008:00100000:1.0:1788880564.349748:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880564.349774:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511764736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880564.349793:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880564.349798:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880564.349800:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880564.349808:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5551 00000100:00080000:1.0:1788880564.349813:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880564.349819:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880564.349820:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880564.349837:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880564.349855:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511764864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880564.349871:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880564/real 1788880564] req@ffffa06188699f80 x1875771511764608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880564.349873:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880564.349891:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880564.349899:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880564.349916:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5551 00000100:00080000:1.0:1788880564.349923:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880564.349929:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:0.0:1788880564.349934:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511764608/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788880564.349940:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880564.349961:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1788880564.349968:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511764992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880564.349972:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:1.0:1788880564.349991:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788880564.350000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511764608:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880564.350002:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880564.350012:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880564.350019:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5551 00000100:00080000:1.0:1788880564.350026:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880564.350030:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880564.350039:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880564.350043:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:1.0:1788880564.350062:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511765120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880564.350081:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880564.350101:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880564/real 1788880564] req@ffffa06188698000 x1875771511764736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880564.350113:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511764736/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880564.350120:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880564.350123:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880564.350126:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764736:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880564.350131:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764864:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880564.350142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880564.350149:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880564/real 1788880564] req@ffffa0618869a680 x1875771511764864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880564.350156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511764864/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880564.350162:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880564.350163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880564.350165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764864:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880564.350170:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764992:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880564.350182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880564.350188:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880564/real 1788880564] req@ffffa0618869bb80 x1875771511764992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880564.350195:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511764992/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880564.350200:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880564.350202:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880564.350205:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511764992:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880564.350219:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765120:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880564.350229:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880564.350236:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880564/real 1788880564] req@ffffa0618869ad80 x1875771511765120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880564.350243:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511765120/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880564.350260:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880564.350263:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880564.350265:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765120:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880564.989504:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880564.989588:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880565.245481:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880566.013542:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880566.013586:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880566.269516:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880567.037491:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880567.037527:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880567.293514:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880568.061525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880568.061534:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880568.317463:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880569.085515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880569.085577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880569.341447:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880569.469582:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880569.469593:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880569.469598:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880569.469613:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5556 00000100:00080000:3.0:1788880569.469626:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880569.469636:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880569.469682:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511765248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880569.469732:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880569.469737:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880569.469740:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880569.469745:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5556 00000100:00080000:3.0:1788880569.469750:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880569.469755:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880569.469780:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880569.469800:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511765248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880569.469803:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511765376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880569.469829:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880569.469833:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880569.469836:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880569.469843:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5556 00000100:00080000:3.0:1788880569.469847:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880569.469852:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880569.469860:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880569.469877:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511765504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880569.469890:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880569.469895:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880569.469897:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880569.469903:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5556 00000100:00080000:3.0:1788880569.469908:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880569.469912:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880569.469918:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880569.469934:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511765632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880569.469944:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880569.469949:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880569.469953:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880569.469958:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765376:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788880569.469966:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5556 00000100:00080000:3.0:1788880569.469975:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880569.469982:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880569.469991:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880569.470013:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511765760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880569.470018:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880569.470086:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880569.470124:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880570.109530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880570.109642:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880570.365496:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880571.133538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880571.133570:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880571.389500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880572.157513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880572.157561:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880572.413499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880572.541702:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880572.541754:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880572.541775:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880572.541821:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880572.541834:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880572.541851:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880572.541869:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:0.0:1788880572.541889:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880569/real 1788880572] req@ffffa06188870700 x1875771511765376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880572.541929:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511765376/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880572.541947:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880572.541957:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880572.541966:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765376:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880572.541982:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880569/real 1788880572] req@ffffa06188872680 x1875771511765504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880572.542008:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511765504/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880572.542026:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880572.542034:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880572.542043:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765504:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880572.542060:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880569/real 1788880572] req@ffffa06188873100 x1875771511765632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880572.542091:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511765632/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880572.542109:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880572.542117:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880572.542126:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765632:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880572.542140:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880569/real 1788880572] req@ffffa06188871500 x1875771511765248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880572.542166:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511765248/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880572.542193:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880572.542228:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880572.542234:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511765248:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880572.542272:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880569/real 1788880572] req@ffffa06188873480 x1875771511765760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880572.542289:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511765760/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880572.542299:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880572.542304:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880572.542308:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511765760:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880573.181533:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880573.181675:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880573.437507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880574.205527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880574.205555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880574.461499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880574.589537:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880574.589547:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880574.589554:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880574.589569:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5561 00000100:00080000:3.0:1788880574.589584:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880574.589595:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880574.589640:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511765888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880574.589675:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880574.589680:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880574.589683:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880574.589688:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5561 00000100:00080000:3.0:1788880574.589692:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880574.589698:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880574.589711:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880574.589736:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511766016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880574.589738:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511765888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880574.589765:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880574.589770:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880574.589773:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880574.589780:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5561 00000100:00080000:3.0:1788880574.589785:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880574.589790:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880574.589798:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788880574.589835:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880574.589878:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880574/real 1788880574] req@ffffa06188870a80 x1875771511765888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880574.589915:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511765888/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880574.589933:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511766144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880574.589950:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880574.589955:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880574.589962:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1788880574.589968:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00080000:3.0:1788880574.589969:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880574.589989:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5561 00000100:00100000:0.0:1788880574.589991:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511765888:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880574.589996:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880574.590012:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880574.590021:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880574.590037:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511766272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880574.590048:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766016:192.168.203.111@tcp:38:kworker.0 00000100:00080000:3.0:1788880574.590056:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880574.590066:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880574.590068:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880574.590075:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5561 00000100:00080000:3.0:1788880574.590080:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880574.590087:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880574.590096:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880574.590126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880574.590127:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511766400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880574.590164:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880574/real 1788880574] req@ffffa06188872d80 x1875771511766016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880574.590220:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511766016/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880574.590234:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880574.590239:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880574.590243:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766016:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880574.590252:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766144:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880574.590283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880574.590298:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880574/real 1788880574] req@ffffa06188872300 x1875771511766144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880574.590313:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511766144/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880574.590323:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880574.590327:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880574.590331:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766144:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880574.590351:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766272:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880574.590372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880574.590418:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880574/real 1788880574] req@ffffa06188873b80 x1875771511766272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880574.590435:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511766272/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880574.590445:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880574.590449:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880574.590453:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766272:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880574.590460:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766400:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880574.590483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880574.590496:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880574/real 1788880574] req@ffffa06188872a00 x1875771511766400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880574.590510:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511766400/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880574.590519:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880574.590523:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880574.590526:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766400:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880575.229513:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880575.229549:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880575.485492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880576.253523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880576.253656:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880576.509492:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880577.277519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880577.277561:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880577.533457:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880578.301569:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880578.301615:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880578.557510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880579.325514:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880579.325675:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880579.581491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880579.709603:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880579.709613:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880579.709618:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880579.709631:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5566 00000100:00080000:1.0:1788880579.709644:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880579.709653:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880579.709701:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869ad80 x1875771511766528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880579.709741:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880579.709747:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880579.709749:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880579.709755:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5566 00000100:00080000:1.0:1788880579.709760:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880579.709765:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880579.709778:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880579.709800:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869bb80 x1875771511766656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880579.709803:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511766528:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880579.709832:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880579.709837:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880579.709839:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880579.709845:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5566 00000100:00080000:1.0:1788880579.709851:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880579.709856:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880579.709863:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880579.709880:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869a680 x1875771511766784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880579.709893:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880579.709898:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880579.709900:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880579.709904:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5566 00000100:00080000:1.0:1788880579.709910:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880579.709911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880579.709920:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880579.709931:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880579.709947:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698000 x1875771511766912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880579.709959:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880579.709967:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788880579.709970:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880579/real 1788880579] req@ffffa0618869ad80 x1875771511766528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880579.709972:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880579.709987:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5566 00000100:00080000:1.0:1788880579.709999:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880579.710009:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880579.710016:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880579.710024:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869ad80 x1875771511766528/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880579.710035:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699f80 x1875771511767040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880579.710052:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880579.710069:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880579.710083:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511766528:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880579.710118:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766656:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880579.710145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880579.710185:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880579/real 1788880579] req@ffffa0618869bb80 x1875771511766656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880579.710202:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869bb80 x1875771511766656/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880579.710213:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880579.710217:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880579.710222:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766656:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880579.710229:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766784:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880579.710251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880579.710264:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880579/real 1788880579] req@ffffa0618869a680 x1875771511766784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880579.710279:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869a680 x1875771511766784/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880579.710289:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880579.710293:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880579.710296:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766784:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880579.710304:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766912:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880579.710323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880579.710334:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880579/real 1788880579] req@ffffa06188698000 x1875771511766912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880579.710348:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698000 x1875771511766912/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880579.710358:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880579.710361:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880579.710365:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511766912:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880579.710371:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767040:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880579.710421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880579.710435:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880579/real 1788880579] req@ffffa06188699f80 x1875771511767040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880579.710449:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699f80 x1875771511767040/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880579.710468:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880579.710472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880579.710476:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767040:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880580.349448:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880580.349565:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880580.605536:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880581.373527:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880581.373573:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880581.629508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880582.397538:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880582.397658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880582.653470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880583.421492:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880583.421558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880583.677472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880584.445495:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880584.445551:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880584.701497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880584.829574:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880584.829587:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880584.829594:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880584.829613:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5571 00000100:00080000:3.0:1788880584.829629:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880584.829642:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880584.829708:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511767168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880584.829752:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880584.829758:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880584.829761:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880584.829767:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5571 00000100:00080000:3.0:1788880584.829773:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880584.829780:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880584.829795:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880584.829819:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511767168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880584.829821:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511767296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880584.829860:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880584.829867:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880584.829870:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880584.829879:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5571 00000100:00080000:3.0:1788880584.829883:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880584.829888:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880584.829896:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880584.829913:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511767424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880584.829927:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880584.829933:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880584.829937:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880584.829947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880584.829955:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5571 00000100:00080000:3.0:1788880584.829962:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880584.829970:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880584.829981:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880584.830002:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511767552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880584.830019:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880584/real 1788880584] req@ffffa06188871c00 x1875771511767168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880584.830023:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880584.830042:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880584.830048:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880584.830068:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5571 00000100:00080000:3.0:1788880584.830079:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880584.830091:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880584.830100:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880584.830104:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511767168/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880584.830121:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511767680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880584.830147:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880584.830166:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880584.830189:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511767168:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880584.830236:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767296:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880584.830282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880584.830301:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880584/real 1788880584] req@ffffa06188873800 x1875771511767296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880584.830318:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511767296/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880584.830330:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880584.830335:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880584.830339:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767296:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880584.830347:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767424:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880584.830369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880584.830414:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880584/real 1788880584] req@ffffa06188871880 x1875771511767424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880584.830431:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511767424/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880584.830442:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880584.830446:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880584.830449:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767424:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880584.830458:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767552:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880584.830483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880584.830497:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880584/real 1788880584] req@ffffa06188870380 x1875771511767552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880584.830511:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511767552/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880584.830522:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880584.830525:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880584.830530:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767552:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880584.830538:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767680:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880584.830557:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880584.830568:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880584/real 1788880584] req@ffffa06188872a00 x1875771511767680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880584.830582:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511767680/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880584.830592:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880584.830595:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880584.830599:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767680:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880585.469534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880585.469700:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880585.725483:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880586.493537:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880586.493537:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880586.749474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880587.517510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880587.517574:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880587.773480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880588.541519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880588.541620:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880588.797499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880589.565521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880589.565540:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880589.821479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880589.949523:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880589.949533:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880589.949539:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880589.949558:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5576 00000100:00080000:3.0:1788880589.949578:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880589.949591:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880589.949650:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511767808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880589.949690:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880589.949695:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880589.949698:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880589.949703:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5576 00000100:00080000:3.0:1788880589.949708:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880589.949714:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880589.949729:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880589.949749:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511767936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880589.949754:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511767808:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880589.949795:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880589.949802:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880589.949806:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880589.949812:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5576 00000100:00080000:3.0:1788880589.949816:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880589.949823:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880589.949830:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880589.949847:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511768064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880589.949862:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880589.949870:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1788880589.949873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880589.949876:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880589.949885:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5576 00000100:00080000:3.0:1788880589.949892:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880589.949900:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880589.949909:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880589.949928:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511768192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880589.949936:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880589/real 1788880589] req@ffffa06188873b80 x1875771511767808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880589.949991:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880589.950010:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880589.950013:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880589.950025:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5576 00000100:00100000:0.0:1788880589.950027:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511767808/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880589.950036:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880589.950057:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788880589.950072:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1788880589.950074:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880589.950126:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880589.950136:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511768320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880589.950147:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511767808:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880589.950213:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767936:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880589.950248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880589.950265:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880589/real 1788880589] req@ffffa06188872300 x1875771511767936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880589.950282:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511767936/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880589.950293:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880589.950297:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880589.950301:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511767936:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880589.950310:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768064:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880589.950329:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880589.950340:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880589/real 1788880589] req@ffffa06188872d80 x1875771511768064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880589.950354:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511768064/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880589.950364:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880589.950367:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880589.950371:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768064:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880589.950378:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768192:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880589.950436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880589.950452:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880589/real 1788880589] req@ffffa06188870a80 x1875771511768192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880589.950467:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511768192/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880589.950477:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880589.950481:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880589.950485:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768192:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880589.950492:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768320:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880589.950511:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880589.950522:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880589/real 1788880589] req@ffffa06188873480 x1875771511768320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880589.950536:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511768320/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880589.950546:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880589.950549:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880589.950553:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768320:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880590.589482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880590.589523:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880590.845465:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880591.613470:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880591.613600:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880591.869474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880592.637478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880592.637506:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880592.893449:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880593.661511:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880593.661546:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880593.917435:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880594.685445:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880594.685595:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880594.941518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880595.069471:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880595.069485:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880595.069492:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880595.069510:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5581 00000100:00080000:3.0:1788880595.069530:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880595.069559:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880595.069606:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871500 x1875771511768448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880595.069630:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880595.069633:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880595.069635:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880595.069638:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5581 00000100:00080000:3.0:1788880595.069641:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880595.069645:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880595.069656:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880595.069666:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511768448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880595.069668:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873100 x1875771511768576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880595.069681:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880595.069684:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880595.069685:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880595.069688:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5581 00000100:00080000:3.0:1788880595.069691:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880595.069693:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880595.069697:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880595.069705:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872680 x1875771511768704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880595.069711:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880595.069714:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880595.069715:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880595.069718:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5581 00000100:00080000:3.0:1788880595.069721:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880595.069723:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880595.069726:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880595.069734:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870700 x1875771511768832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880595.069739:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880595.069742:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880595.069743:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880595.069746:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5581 00000100:00080000:3.0:1788880595.069749:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880595.069751:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880595.069757:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788880595.069758:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880595.069771:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511768960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880595.069810:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880595/real 1788880595] req@ffffa06188871500 x1875771511768448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880595.069847:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871500 x1875771511768448/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880595.069862:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880595.069868:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880595.069883:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511768448:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880595.069918:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768576:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880595.069950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880595.069965:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880595/real 1788880595] req@ffffa06188873100 x1875771511768576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880595.069981:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873100 x1875771511768576/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880595.069992:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880595.069996:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880595.070000:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768576:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880595.070007:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768704:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880595.070026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880595.070037:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880595/real 1788880595] req@ffffa06188872680 x1875771511768704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880595.070052:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872680 x1875771511768704/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880595.070124:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880595.070127:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880595.070129:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768704:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880595.070133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768832:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880595.070149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880595.070157:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880595/real 1788880595] req@ffffa06188870700 x1875771511768832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880595.070202:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870700 x1875771511768832/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880595.070224:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880595.070230:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880595.070237:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768832:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880595.070249:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768960:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880595.070291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880595.070314:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880595/real 1788880595] req@ffffa06188873480 x1875771511768960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880595.070330:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511768960/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880595.070341:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880595.070346:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880595.070351:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511768960:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880595.709459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880595.709466:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880595.965502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880596.733482:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880596.733501:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880596.989479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880597.757496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880597.757661:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880598.013456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880598.781488:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880598.781488:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880599.037498:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880599.805478:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880599.805522:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880600.061454:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880600.189440:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880600.189446:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880600.189451:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880600.189482:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5586 00000100:00080000:3.0:1788880600.189499:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880600.189511:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880600.189568:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511769088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880600.189608:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880600.189613:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880600.189616:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880600.189622:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5586 00000100:00080000:3.0:1788880600.189627:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880600.189633:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880600.189649:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880600.189667:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511769088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880600.189670:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511769216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880600.189693:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880600.189699:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880600.189701:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880600.189707:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5586 00000100:00080000:3.0:1788880600.189713:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880600.189717:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880600.189723:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880600.189739:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511769344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880600.189751:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880600.189756:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880600.189759:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880600.189764:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5586 00000100:00080000:3.0:1788880600.189768:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880600.189772:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880600.189778:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1788880600.189780:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880600.189811:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511769472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880600.189819:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880600.189822:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880600.189824:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880600.189826:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5586 00000100:00080000:3.0:1788880600.189829:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880600.189832:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880600.189837:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880600.189841:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880600/real 1788880600] req@ffffa06188870a80 x1875771511769088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880600.189849:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511769600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880600.189863:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511769088/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880600.189890:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880600.189898:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880600.189903:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511769088:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880600.189936:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769216:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880600.189967:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880600.189982:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880600/real 1788880600] req@ffffa06188872d80 x1875771511769216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880600.189998:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511769216/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880600.190009:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880600.190014:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880600.190019:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769216:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880600.190027:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769344:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880600.190097:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880600.190106:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880600/real 1788880600] req@ffffa06188872300 x1875771511769344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880600.190114:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511769344/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880600.190120:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880600.190122:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880600.190125:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769344:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880600.190130:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769472:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880600.190142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880600.190148:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880600/real 1788880600] req@ffffa06188873b80 x1875771511769472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880600.190156:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511769472/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880600.190161:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880600.190163:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880600.190165:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769472:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880600.190169:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769600:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880600.190179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880600.190186:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880600/real 1788880600] req@ffffa06188872a00 x1875771511769600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880600.190193:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511769600/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880600.190199:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880600.190201:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880600.190202:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769600:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880600.829518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880600.829651:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880601.085485:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880601.853521:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880601.853558:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880602.109510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880602.877529:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880602.877567:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880603.133503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880603.901507:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880603.901637:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880604.157499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880604.925499:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880604.925551:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880605.181490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880605.309568:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880605.309579:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880605.309586:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880605.309603:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5592 00000100:00080000:3.0:1788880605.309619:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880605.309631:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880605.309685:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870380 x1875771511769728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880605.309722:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880605.309727:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880605.309730:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880605.309735:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5592 00000100:00080000:3.0:1788880605.309740:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880605.309746:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880605.309761:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880605.309785:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871880 x1875771511769856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880605.309787:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511769728:192.168.203.111@tcp:250:kworker.0 00000100:00080000:3.0:1788880605.309816:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880605.309821:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880605.309824:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880605.309828:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5592 00000100:00080000:3.0:1788880605.309832:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880605.309837:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880605.309844:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880605.309862:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873800 x1875771511769984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880605.309874:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880605.309879:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880605.309881:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880605.309886:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5592 00000100:00080000:3.0:1788880605.309891:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880605.309896:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880605.309902:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880605.309922:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188871c00 x1875771511770112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788880605.309926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:3.0:1788880605.309944:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880605.309950:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880605.309953:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880605.309959:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5592 00000100:00080000:3.0:1788880605.309964:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880605.309969:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880605.309975:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880605.309990:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872a00 x1875771511770240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880605.310030:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880605/real 1788880605] req@ffffa06188870380 x1875771511769728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880605.310062:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870380 x1875771511769728/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880605.310077:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880605.310083:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880605.310089:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511769728:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880605.310123:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769856:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880605.310158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880605.310176:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880605/real 1788880605] req@ffffa06188871880 x1875771511769856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880605.310191:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871880 x1875771511769856/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880605.310202:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880605.310205:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880605.310210:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769856:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880605.310217:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769984:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880605.310237:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880605.310249:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880605/real 1788880605] req@ffffa06188873800 x1875771511769984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880605.310263:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873800 x1875771511769984/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880605.310273:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880605.310276:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880605.310280:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511769984:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880605.310286:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770112:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880605.310306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880605.310317:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880605/real 1788880605] req@ffffa06188871c00 x1875771511770112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880605.310331:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188871c00 x1875771511770112/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880605.310342:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880605.310346:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880605.310350:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188871c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770112:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880605.310356:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770240:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880605.310373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880605.310417:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880605/real 1788880605] req@ffffa06188872a00 x1875771511770240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880605.310433:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872a00 x1875771511770240/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880605.310443:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880605.310447:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880605.310451:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770240:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880605.949532:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880605.949544:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880606.205499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880606.973502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880606.973658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880607.229491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880607.997494:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880607.997555:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880608.253504:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880609.021531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880609.021553:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880609.277474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880610.045531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880610.045666:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880610.301509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:3.0:1788880610.429509:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880610.429519:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1788880610.429526:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880610.429541:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5597 00000100:00080000:3.0:1788880610.429557:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880610.429571:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:3.0:1788880610.429630:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873b80 x1875771511770368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880610.429667:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880610.429673:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1788880610.429675:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880610.429681:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5597 00000100:00080000:3.0:1788880610.429685:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880610.429692:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880610.429708:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880610.429729:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511770368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880610.429731:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872300 x1875771511770496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880610.429758:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880610.429763:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1788880610.429765:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880610.429771:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5597 00000100:00080000:3.0:1788880610.429775:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880610.429779:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880610.429784:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788880610.429804:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188872d80 x1875771511770624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880610.429816:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880610.429820:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1788880610.429823:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880610.429828:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5597 00000100:00080000:3.0:1788880610.429833:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880610.429839:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880610.429847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:3.0:1788880610.429850:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880610.429871:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188870a80 x1875771511770752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880610.429884:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788880610.429889:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1788880610.429892:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788880610.429899:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5597 00000100:00100000:0.0:1788880610.429906:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880610/real 1788880610] req@ffffa06188873b80 x1875771511770368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880610.429908:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788880610.429922:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:3.0:1788880610.429938:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1788880610.429954:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873b80 x1875771511770368/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880610.429962:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188873480 x1875771511770880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788880610.430003:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880610.430016:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880610.430024:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511770368:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880610.430059:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770496:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880610.430089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880610.430105:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880610/real 1788880610] req@ffffa06188872300 x1875771511770496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880610.430122:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872300 x1875771511770496/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880610.430133:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880610.430137:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880610.430142:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770496:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880610.430149:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770624:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880610.430168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880610.430180:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880610/real 1788880610] req@ffffa06188872d80 x1875771511770624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880610.430194:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188872d80 x1875771511770624/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880610.430204:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880610.430207:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880610.430211:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188872d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770624:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880610.430219:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770752:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880610.430239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880610.430251:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880610/real 1788880610] req@ffffa06188870a80 x1875771511770752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880610.430265:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188870a80 x1875771511770752/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880610.430275:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880610.430278:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880610.430282:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188870a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770752:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880610.430289:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770880:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880610.430308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880610.430319:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880610/real 1788880610] req@ffffa06188873480 x1875771511770880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880610.430333:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188873480 x1875771511770880/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880610.430343:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880610.430346:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880610.430349:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188873480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511770880:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880611.069534:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880611.069559:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880611.325472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880612.093505:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880612.093556:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880612.349490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880613.117536:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880613.117659:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880613.373480:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:0.0:1788880614.141500:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880614.141653:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880614.397470:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788880615.165518:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880615.165616:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788880615.421505:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880615.549587:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880615.549598:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880615.549603:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880615.549617:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5602 00000100:00080000:2.0:1788880615.549632:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880615.549641:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880615.549690:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac000 x1875771511771008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880615.549730:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880615.549736:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880615.549738:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880615.549744:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5602 00000100:00080000:2.0:1788880615.549748:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880615.549754:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880615.549768:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880615.549789:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad500 x1875771511771136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880615.549792:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511771008:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880615.549819:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880615.549823:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880615.549826:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880615.549830:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5602 00000100:00080000:2.0:1788880615.549835:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880615.549839:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880615.549845:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880615.549862:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af100 x1875771511771264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880615.549874:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880615.549879:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880615.549882:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880615.549886:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5602 00000100:00080000:2.0:1788880615.549891:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880615.549896:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000800:00000100:0.0:1788880615.549906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000008:00100000:2.0:1788880615.549909:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880615.549928:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888af480 x1875771511771392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880615.549964:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880615.549969:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880615.549972:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880615.549979:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5602 00000100:00080000:2.0:1788880615.549985:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788880615.549987:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880615/real 1788880615] req@ffffa061888ac000 x1875771511771008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880615.549995:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880615.550007:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880615.550032:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888afb80 x1875771511771520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880615.550042:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac000 x1875771511771008/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880615.550068:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880615.550074:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880615.550083:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511771008:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880615.550114:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771136:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880615.550144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880615.550159:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880615/real 1788880615] req@ffffa061888ad500 x1875771511771136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880615.550175:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad500 x1875771511771136/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880615.550186:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880615.550190:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880615.550194:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771136:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880615.550201:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771264:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880615.550220:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880615.550232:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880615/real 1788880615] req@ffffa061888af100 x1875771511771264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880615.550256:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af100 x1875771511771264/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880615.550266:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880615.550269:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880615.550273:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771264:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880615.550280:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771392:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880615.550300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880615.550311:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880615/real 1788880615] req@ffffa061888af480 x1875771511771392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880615.550326:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888af480 x1875771511771392/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880615.550336:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880615.550339:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880615.550343:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771392:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880615.550349:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771520:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880615.550367:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880615.550377:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880615/real 1788880615] req@ffffa061888afb80 x1875771511771520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880615.550423:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888afb80 x1875771511771520/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880615.550441:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880615.550445:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880615.550449:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771520:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880616.189547:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880616.189647:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880616.445487:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880617.213525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880617.213545:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880617.469503:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880618.237501:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880618.237558:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880618.493506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880619.261528:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880619.261617:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880619.517495:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880620.285530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880620.285566:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880620.541500:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880620.669539:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880620.669550:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880620.669557:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880620.669576:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5607 00000100:00080000:2.0:1788880620.669593:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880620.669605:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880620.669658:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adf80 x1875771511771648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880620.669694:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880620.669700:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880620.669703:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880620.669708:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5607 00000100:00080000:2.0:1788880620.669713:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880620.669719:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880620.669732:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880620.669749:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aca80 x1875771511771776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880620.669770:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511771648:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880620.669771:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880620.669786:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880620.669794:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880620.669802:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5607 00000100:00080000:2.0:1788880620.669805:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880620.669809:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880620.669816:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880620.669833:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511771904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880620.669847:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880620.669852:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880620.669855:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880620.669861:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5607 00000100:00080000:2.0:1788880620.669865:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880620.669870:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880620.669877:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880620.669891:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511772032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880620.669902:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880620.669936:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880620.669949:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880620.669950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880620.669970:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5607 00000100:00080000:2.0:1788880620.669980:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880620.669988:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880620.669996:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880620.670018:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511772160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880620.670036:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880620/real 1788880620] req@ffffa061888adf80 x1875771511771648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880620.670070:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adf80 x1875771511771648/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880620.670086:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880620.670092:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880620.670097:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511771648:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880620.670133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771776:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880620.670169:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880620.670185:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880620/real 1788880620] req@ffffa061888aca80 x1875771511771776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880620.670201:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aca80 x1875771511771776/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880620.670212:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880620.670216:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880620.670220:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771776:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880620.670229:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771904:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880620.670249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880620.670260:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880620/real 1788880620] req@ffffa061888ae300 x1875771511771904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880620.670275:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511771904/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880620.670284:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880620.670288:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880620.670292:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511771904:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880620.670300:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772032:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880620.670321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880620.670332:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880620/real 1788880620] req@ffffa061888aea00 x1875771511772032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880620.670346:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511772032/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880620.670356:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880620.670359:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880620.670363:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772032:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880620.670370:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772160:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880620.670427:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880620.670441:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880620/real 1788880620] req@ffffa061888adc00 x1875771511772160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880620.670457:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511772160/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880620.670468:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880620.670472:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880620.670476:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772160:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880621.309550:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880621.309594:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880621.565510:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880622.333525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880622.333664:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880622.589508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880623.357539:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880623.357545:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880623.613539:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00010000:00010000:3.0:1788880624.381525:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1788880624.381530:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00000100:00100000:1.0:1788880624.637512:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880625.405512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880625.405658:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880625.661507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880625.789520:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880625.789529:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880625.789535:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880625.789548:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5612 00000100:00080000:2.0:1788880625.789562:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880625.789572:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880625.789619:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511772288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880625.789655:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880625.789660:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880625.789663:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880625.789669:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5612 00000100:00080000:2.0:1788880625.789674:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880625.789679:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880625.789692:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788880625.789713:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511772288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:2.0:1788880625.789713:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511772416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880625.789749:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880625.789753:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880625.789756:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880625.789762:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5612 00000100:00080000:2.0:1788880625.789765:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880625.789773:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880625.789779:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880625.789795:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511772544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880625.789811:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880625.789816:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880625.789819:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788880625.789830:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:2.0:1788880625.789837:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5612 00000100:00080000:2.0:1788880625.789845:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880625.789850:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880625.789860:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880625.789905:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511772672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880625.789907:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880625/real 1788880625] req@ffffa061888ad880 x1875771511772288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880625.789999:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880625.790005:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880625.790008:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788880625.790016:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511772288/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788880625.790020:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5612 00000100:00080000:2.0:1788880625.790034:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880625.790052:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00080000:0.0:1788880625.790056:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788880625.790059:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1788880625.790070:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:2.0:1788880625.790079:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511772800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880625.790084:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511772288:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880625.790146:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772416:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880625.790176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880625.790193:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880625/real 1788880625] req@ffffa061888ac700 x1875771511772416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880625.790208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511772416/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880625.790227:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880625.790230:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880625.790233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772416:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880625.790239:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772544:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880625.790257:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880625.790265:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880625/real 1788880625] req@ffffa061888ace00 x1875771511772544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880625.790274:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511772544/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880625.790279:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880625.790282:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880625.790284:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772544:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880625.790288:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772672:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880625.790300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880625.790306:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880625/real 1788880625] req@ffffa061888ac380 x1875771511772672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880625.790313:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511772672/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880625.790319:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880625.790321:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880625.790323:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772672:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880625.790328:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772800:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880625.790340:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880625.790345:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880625/real 1788880625] req@ffffa061888aed80 x1875771511772800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880625.790353:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511772800/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880625.790359:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880625.790360:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880625.790362:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511772800:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880626.429512:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880626.429512:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880626.685472:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880627.453545:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880627.453604:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880627.709526:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880628.477509:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880628.477646:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880628.733508:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880629.501531:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880629.501542:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880629.757518:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880630.525523:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880630.525587:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880630.781459:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880630.909530:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880630.909537:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880630.909541:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880630.909551:0:34825:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5617 00000100:00080000:1.0:1788880630.909561:0:34825:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880630.909567:0:34825:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880630.909600:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511772928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880630.909625:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880630.909627:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880630.909629:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880630.909632:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5617 00000100:00080000:1.0:1788880630.909634:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880630.909637:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880630.909646:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880630.909656:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511773056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880630.909664:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880630.909669:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1788880630.909671:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511772928:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880630.909671:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880630.909687:0:34825:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5617 00000100:00080000:1.0:1788880630.909689:0:34825:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880630.909692:0:34825:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880630.909695:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880630.909707:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511773184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880630.909713:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880630.909716:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880630.909717:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880630.909720:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5617 00000100:00080000:1.0:1788880630.909722:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880630.909726:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880630.909733:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880630.909739:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511773312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880630.909746:0:34825:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880630.909748:0:34825:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880630.909750:0:34825:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880630.909752:0:34825:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5617 00000100:00080000:1.0:1788880630.909756:0:34825:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788880630.909759:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00080000:1.0:1788880630.909764:0:34825:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880630.909769:0:34825:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1788880630.909783:0:34825:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511773440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788880630.909812:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880630/real 1788880630] req@ffffa0618869b100 x1875771511772928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880630.909828:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511772928/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880630.909837:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880630.909854:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880630.909858:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511772928:192.168.203.111@tcp:250:kworker.0 00000100:00100000:0.0:1788880630.909883:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773056:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880630.909903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880630.909913:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880630/real 1788880630] req@ffffa06188698380 x1875771511773056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880630.909922:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511773056/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880630.909928:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880630.909931:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880630.909933:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773056:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880630.909938:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773184:192.168.203.111@tcp:38:kworker.0 00000800:00000100:0.0:1788880630.909949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880630.909955:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880630/real 1788880630] req@ffffa0618869b800 x1875771511773184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880630.909963:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511773184/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880630.909968:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880630.909970:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880630.909971:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773184:192.168.203.111@tcp:38:kworker.0 00000100:00100000:0.0:1788880630.909976:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773312:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880630.909987:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880630.909993:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880630/real 1788880630] req@ffffa06188699c00 x1875771511773312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880630.910000:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511773312/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880630.910007:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880630.910009:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880630.910010:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773312:192.168.203.111@tcp:8:kworker.0 00000100:00100000:0.0:1788880630.910014:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773440:192.168.203.111@tcp:8:kworker.0 00000800:00000100:0.0:1788880630.910117:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:0.0:1788880630.910140:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880630/real 1788880630] req@ffffa0618869aa00 x1875771511773440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788880630.910154:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511773440/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788880630.910162:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788880630.910165:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:0.0:1788880630.910168:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773440:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880631.549480:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880631.549663:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880631.805491:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880632.573463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880632.573541:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880632.829507:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880633.597489:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880633.597569:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880633.853474:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880634.621442:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880634.621602:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880634.877523:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880635.645485:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880635.645530:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880635.901479:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880636.029509:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880636.029518:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880636.029521:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880636.029531:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5622 00000100:00080000:2.0:1788880636.029541:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880636.029548:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880636.029582:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae680 x1875771511773568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880636.029613:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880636.029616:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880636.029617:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880636.029621:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5622 00000100:00080000:2.0:1788880636.029623:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880636.029625:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880636.029634:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880636.029649:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad180 x1875771511773696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880636.029658:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880636.029661:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880636.029662:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880636.029664:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5622 00000100:00080000:2.0:1788880636.029666:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880636.029668:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880636.029671:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880636.029684:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aed80 x1875771511773824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880636.029691:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511773568:192.168.203.111@tcp:250:kworker.0 00000100:00080000:2.0:1788880636.029697:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880636.029703:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880636.029705:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880636.029708:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5622 00000100:00080000:2.0:1788880636.029710:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880636.029713:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880636.029718:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880636.029727:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac380 x1875771511773952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880636.029734:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880636.029737:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880636.029739:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880636.029742:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5622 00000100:00080000:2.0:1788880636.029744:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880636.029747:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880636.029750:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880636.029757:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ace00 x1875771511774080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880636.029836:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880636.029893:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880636.029917:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880636.029942:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880636.669519:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880636.669577:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880636.925494:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880637.693525:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880637.693739:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880637.949524:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880638.717490:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880638.717605:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880638.973490:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000400:00000100:1.0:1788880639.101758:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.111:988 00000400:00000100:1.0:1788880639.101852:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.111@tcp at host 192.168.203.111 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788880639.101869:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880639.101911:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880639.101920:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880639.101929:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000800:00000100:1.0:1788880639.101937:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.111@tcp 00000100:00100000:3.0:1788880639.102038:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880636/real 1788880639] req@ffffa061888aed80 x1875771511773824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880639.102070:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aed80 x1875771511773824/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880639.102084:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880639.102090:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880639.102094:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773824:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880639.102104:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880636/real 1788880639] req@ffffa061888ac380 x1875771511773952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880639.102129:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac380 x1875771511773952/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880639.102137:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880639.102141:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880639.102144:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773952:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880639.102152:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880636/real 1788880639] req@ffffa061888ace00 x1875771511774080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880639.102164:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ace00 x1875771511774080/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880639.102172:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880639.102175:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880639.102179:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774080:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880639.102196:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880636/real 1788880639] req@ffffa061888ae680 x1875771511773568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880639.102208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae680 x1875771511773568/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880639.102216:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880639.102220:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880639.102223:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511773568:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880639.102230:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880636/real 1788880639] req@ffffa061888ad180 x1875771511773696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880639.102243:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad180 x1875771511773696/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880639.102251:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880639.102254:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880639.102258:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511773696:192.168.203.111@tcp:38:kworker.0 00000100:00100000:2.0:1788880639.741496:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880639.741560:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880639.997499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880640.765502:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880640.765581:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880641.021522:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:2.0:1788880641.149542:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880641.149553:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1788880641.149560:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880641.149574:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5627 00000100:00080000:2.0:1788880641.149589:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880641.149600:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:2.0:1788880641.149637:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ac700 x1875771511774208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880641.149663:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880641.149666:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1788880641.149668:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880641.149672:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5627 00000100:00080000:2.0:1788880641.149674:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880641.149677:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880641.149687:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880641.149696:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ad880 x1875771511774336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880641.149703:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880641.149705:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1788880641.149706:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880641.149708:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5627 00000100:00080000:2.0:1788880641.149710:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880641.149714:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880641.149717:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788880641.149723:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888adc00 x1875771511774464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880641.149731:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880641.149734:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1788880641.149736:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880641.149738:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5627 00000100:00080000:2.0:1788880641.149740:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880641.149742:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880641.149745:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880641.149752:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888aea00 x1875771511774592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788880641.149757:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788880641.149759:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788880641.149761:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788880641.149763:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5627 00000100:00080000:2.0:1788880641.149765:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1788880641.149766:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:2.0:1788880641.149769:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1788880641.149774:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa061888ae300 x1875771511774720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880641.149922:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511774208:192.168.203.111@tcp:250:kworker.0 00000800:00000100:3.0:1788880641.150007:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880641.150064:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880641/real 1788880641] req@ffffa061888ac700 x1875771511774208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880641.150087:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ac700 x1875771511774208/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880641.150100:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880641.150122:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880641.150126:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511774208:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880641.150133:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774336:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880641.150150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880641.150159:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880641/real 1788880641] req@ffffa061888ad880 x1875771511774336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880641.150168:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ad880 x1875771511774336/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880641.150174:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880641.150176:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880641.150178:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774336:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880641.150182:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774464:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880641.150193:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880641.150200:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880641/real 1788880641] req@ffffa061888adc00 x1875771511774464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880641.150208:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888adc00 x1875771511774464/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880641.150213:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880641.150215:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880641.150233:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774464:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880641.150238:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774592:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880641.150250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880641.150256:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880641/real 1788880641] req@ffffa061888aea00 x1875771511774592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880641.150264:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888aea00 x1875771511774592/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880641.150269:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880641.150276:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880641.150278:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774592:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880641.150282:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774720:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880641.150293:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880641.150300:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880641/real 1788880641] req@ffffa061888ae300 x1875771511774720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880641.150307:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa061888ae300 x1875771511774720/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880641.150313:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880641.150315:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880641.150317:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa061888ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774720:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880641.789483:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:3.0:1788880641.789516:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880642.045456:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880642.813479:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880642.813547:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880643.069493:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880643.837463:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880643.837550:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880644.093475:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880644.861466:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:2.0:1788880644.861503:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880645.117506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880645.885522:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880645.885710:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880646.141502:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00080000:1.0:1788880646.269587:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880646.269596:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1788880646.269601:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880646.269615:0:29258:0:(import.c:545:import_select_connection()) MGC192.168.203.111@tcp: connect to NID 192.168.203.111@tcp last attempt 5632 00000100:00080000:1.0:1788880646.269628:0:29258:0:(import.c:630:import_select_connection()) MGC192.168.203.111@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880646.269638:0:29258:0:(import.c:665:import_select_connection()) MGC192.168.203.111@tcp: import ffffa061854d7000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000100:00100000:1.0:1788880646.269686:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869aa00 x1875771511774848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880646.269725:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880646.269730:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1788880646.269733:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880646.269738:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5632 00000100:00080000:1.0:1788880646.269743:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880646.269773:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0000-mdc-ffffa06189393000: import ffffa06183c27000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880646.269787:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880646.269815:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188699c00 x1875771511774976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788880646.269832:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511774848:192.168.203.111@tcp:250:kworker.0 00000100:00080000:1.0:1788880646.269837:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880646.269848:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1788880646.269851:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880646.269857:0:29258:0:(import.c:545:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5632 00000100:00080000:1.0:1788880646.269861:0:29258:0:(import.c:630:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880646.269866:0:29258:0:(import.c:665:import_select_connection()) lustre-MDT0001-mdc-ffffa06189393000: import ffffa061893a2000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880646.269872:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683394 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788880646.269900:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b800 x1875771511775104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880646.269913:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880646.269918:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788880646.269921:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788880646.269926:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5632 00000100:00080000:1.0:1788880646.269932:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880646.269936:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0000-osc-ffffa06189393000: import ffffa061893a3000 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880646.269942:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1788880646.269946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:1.0:1788880646.269964:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa06188698380 x1875771511775232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788880646.269988:0:29258:0:(pinger.c:229:ptlrpc_pinger_process_import()) f0204769-56bd-4f5c-ab97-74c450c9e072->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788880646.269993:0:29258:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788880646.269997:0:29258:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788880646.270000:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880646/real 1788880646] req@ffffa0618869aa00 x1875771511774848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788880646.270023:0:29258:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: connect to NID 192.168.203.111@tcp last attempt 5632 00000100:00080000:1.0:1788880646.270030:0:29258:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788880646.270037:0:29258:0:(import.c:665:import_select_connection()) lustre-OST0001-osc-ffffa06189393000: import ffffa061893a2800 using connection 192.168.203.111@tcp/192.168.203.111@tcp 00000008:00100000:1.0:1788880646.270042:0:29258:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683394 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1788880646.270048:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869aa00 x1875771511774848/t0(0) o250->MGC192.168.203.111@tcp@192.168.203.111@tcp:26/25 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788880646.270053:0:29258:0:(import.c:870:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffffa0618869b100 x1875771511775360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788880646.270087:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061854d7000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880646.270093:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880646.270098:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7c1cdf7-2492-4eca-92b8-e6fb71d0eec1:2394:1875771511774848:192.168.203.111@tcp:250:kworker.0 00000100:00100000:3.0:1788880646.270119:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774976:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880646.270138:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880646.270147:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880646/real 1788880646] req@ffffa06188699c00 x1875771511774976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880646.270157:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188699c00 x1875771511774976/t0(0) o38->lustre-MDT0000-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880646.270164:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa06183c27000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880646.270167:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880646.270169:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188699c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511774976:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880646.270173:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511775104:192.168.203.111@tcp:38:kworker.0 00000800:00000100:3.0:1788880646.270185:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880646.270191:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880646/real 1788880646] req@ffffa0618869b800 x1875771511775104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880646.270200:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b800 x1875771511775104/t0(0) o38->lustre-MDT0001-mdc-ffffa06189393000@192.168.203.111@tcp:12/10 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880646.270206:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880646.270208:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880646.270210:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511775104:192.168.203.111@tcp:38:kworker.0 00000100:00100000:3.0:1788880646.270215:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511775232:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880646.270228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880646.270234:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880646/real 1788880646] req@ffffa06188698380 x1875771511775232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880646.270243:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa06188698380 x1875771511775232/t0(0) o8->lustre-OST0000-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880646.270249:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a3000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880646.270251:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880646.270253:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa06188698380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511775232:192.168.203.111@tcp:8:kworker.0 00000100:00100000:3.0:1788880646.270257:0:2394:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511775360:192.168.203.111@tcp:8:kworker.0 00000800:00000100:3.0:1788880646.270294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.111@tcp 00000100:00100000:3.0:1788880646.270305:0:2394:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788880646/real 1788880646] req@ffffa0618869b100 x1875771511775360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788880646.270319:0:2394:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffffa0618869b100 x1875771511775360/t0(0) o8->lustre-OST0001-osc-ffffa06189393000@192.168.203.111@tcp:28/4 lens 520/544 e 0 to 1 dl 1788880671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788880646.270328:0:2394:0:(import.c:65:import_set_state_nolock()) ffffa061893a2800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788880646.270332:0:2394:0:(import.c:1555:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.111@tcp failed (-110) 00000100:00100000:3.0:1788880646.270335:0:2394:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffffa0618869b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:f0204769-56bd-4f5c-ab97-74c450c9e072:2394:1875771511775360:192.168.203.111@tcp:8:kworker.0 00000100:00100000:2.0:1788880646.909518:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880646.909644:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880647.165506:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880647.933459:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880647.933552:0:15191:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880648.189497:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880648.957510:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:0.0:1788880648.957658:0:33372:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880649.213509:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds 00000100:00100000:2.0:1788880649.981515:0:15192:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06190e109c0 going to sleep for 0 seconds 00010000:00010000:1.0:1788880649.981664:0:43370:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1788880650.237499:0:43272:0:(client.c:2716:ptlrpc_set_wait()) set ffffa06185203540 going to sleep for 0 seconds Debug log: 77917 lines, 77917 kept, 0 dropped, 0 bad.